"use strict";

const globals = {};

if (typeof performance === 'undefined') {
  // Fuck, I can't find how to make this browser/node portable more nicely.
  // https://github.com/nodejs/node/issues/28635
  // https://github.com/browserify/perf-hooks-browserify
  //
  // eval('require') because react-scripts build from web/
  // calls webpack, which for some reason cannot find it.
  globals.performance = eval('require')('perf_hooks').performance;
} else {
  globals.performance = performance;
}

const katex = require('katex');
const lodash = require('lodash');
const path = require('path');
const pluralize = require('pluralize');

// consts used by classes.
const HEADER_MENU_ITEM_SEP = '<span class="sep"></span>';
const UNICODE_LINK = '<span class="fa-solid-900">\u{f0c1}</span>';
const NOSPLIT_MARKER_TEXT = 'nosplit'
exports.NOSPLIT_MARKER_TEXT = NOSPLIT_MARKER_TEXT;
const NOSPLIT_MARKER = `<span class="fa-solid-900">\u{f111}</span> ${NOSPLIT_MARKER_TEXT}`;
const SPLIT_MARKER_TEXT = 'split'
exports.SPLIT_MARKER_TEXT = SPLIT_MARKER_TEXT;
const SPLIT_MARKER = `<span class="fa-solid-900">\u{f042}</span> ${SPLIT_MARKER_TEXT}`;
const PARENT_MARKER = '<span class="fa-solid-900">\u{f062}</span>';
exports.PARENT_MARKER = PARENT_MARKER;
const TOC_MARKER = '<span class="fa-solid-900">\u{f03a}</span> toc'

class AstNode {
  /**
   * Abstract syntax tree node. This is the base node type that
   * represents the parsed output.
   *
   * @param {AstType} node_type -
   * @param {String} macro_name - - if node_type === AstType.PLAINTEXT or AstType.ERROR: fixed to
   *                                AstType.PLAINTEXT_MACRO_NAME
   *                              - elif node_type === AstType.PARAGRAPH: fixed to undefined
   *                              - else: arbitrary regular macro
   * @param {Object[String, AstArgument]} args - dict of arg names to arguments.
   *        where arguments are arrays of AstNode
   * @param {SourceLocation} source_location - the best representation of where the macro is starts in the document
   *                        used primarily to present useful debug messages
   * @param {Object} options
   *                 {String} text - the text content of an AstType.PLAINTEXT, undefined for other types
   */
  constructor(node_type, macro_name, args, source_location, options={}) {
    if (!('count_words' in options)) {
      options.count_words = true;
    }
    if (!('first_toplevel_child' in options)) {
      options.first_toplevel_child = false;
    }
    if (!('from_include' in options)) {
      options.from_include = false;
    }
    if (!('from_cirodown_example' in options)) {
      options.from_cirodown_example = false;
    }
    if (!('force_no_index' in options)) {
      options.force_no_index = false;
    }
    if (!(Macro.ID_ARGUMENT_NAME in options)) {
      options.id = undefined;
    }
    if (!('header_parent_ids' in options)) {
      options.header_parent_ids = [];
    }
    if (!('header_tree_node_word_count' in options)) {
      options.header_tree_node_word_count = 0;
    }
    if (!('is_first_header_in_input_file' in options)) {
      // Either the first header on a regular toplevel input,
      // or the first header inside an included include.
      options.is_first_header_in_input_file = false;
    }
    if (!('numbered' in options)) {
      options.numbered = true;
    }
    if (!('parent_node' in options)) {
      // AstNode. This is different from header_tree_node because
      // it points to the parent ast node, i.e. the ast_node that
      // contains this inside one of its arguments.
      //
      // header_tree_node on the other hand points to the header tree.
      // The header tree is currently not even connected via arguments.
      options.parent_node = undefined;
    }
    if (!('split_default' in options)) {
      options.split_default = false;
    }
    if (!('synonym' in options)) {
      options.synonym = undefined;
    }
    if (!('text' in options)) {
      options.text = undefined;
    }
    if (!('xss_safe' in options)) {
      // If true, force this particular instance of the macro to be safe.
      // even if the macro is unsafe by default.
      // This is used for autogenerated elements like incoming links, which.
      // are guaranteed to be safe through other means.
      options.xss_safe = false;
    }

    // Generic fields.
    this.node_type = node_type;
    this.macro_name = macro_name;

    // For elements that have an id.
    // {String} or undefined.
    // Includes scope. Ideally, we should remove that requirement to not duplicate.
    // the information. But it will require a large hard refactor... lazy.
    this.id = options.id;

    // Current running scope. This is inherited from the scope of the ancestor headers.
    // An element with {scope} set does not get this option set (except if due to ancestors),
    // only its children do.
    this.scope = options.scope;

    // For elements that are of AstType.PLAINTEXT.
    this.text = options.text
    this.source_location = source_location;

    this.args = args;
    // The effetctive value for file, coming either from the file=XXX or title if that is empty.
    this.file = undefined
    this.first_toplevel_child = options.first_toplevel_child;
    this.is_first_header_in_input_file = options.is_first_header_in_input_file;
    // This is the Nth macro of this type that appears in the document.
    this.macro_count = undefined;
    // A unique global index for this macro.
    this.macro_count_global = undefined;
    // This is the Nth macro of this type that is visible,
    // and therefore increments counts such as Figure 1), Figure 2), etc.
    // All indexed IDs (those that can be linked to) are also visible, but
    // it is possible to force non-indexed IDs to count as well with
    // caption_number_visible.
    this.macro_count_visible = undefined;
    // Is this header numbered? Note that this is not set based on the numbered= argument:
    // that argument determines this .numbered of descendants only. This is analogous to scope.
    this.numbered = options.numbered;
    // {AstNode} that contains this as an argument.
    this.parent_node = options.parent_node;
    // AstArgument
    this.parent_argument = undefined;
    this.split_default = options.split_default;
    // {HeaderTreeNode} that points to the element.
    // This is used for both headers and non headers:
    // the only difference is that non-headers are not connected as
    // children of their parent. But they still know who the parent is.
    // This was originally required for header scope resolution.
    this.header_tree_node = options.header_tree_node;
    // For DB serialization since we don't current serialize header_tree_node.
    this.header_tree_node_word_count = options.header_tree_node_word_count

    // When fetching Nodes from the database, we only get their ID,
    // so we can't construct a full proper header_tree_node from that alone.
    // So we just store the IDs here and not on header_tree_node as that
    // alone is already useful.
    this.header_parent_ids = options.header_parent_ids;

    this.validation_error = undefined;
    this.validation_output = {};

    // The ID of this element has been indexed.
    this.index_id = undefined;
    this.force_no_index = options.force_no_index;
    // The ToC will go before this header.
    this.toc_header = false;
    // This ast is a descendant of a header. Applications:
    // - header descendants don't count for word counts
    this.in_header = false;
    // {String} the id of the target of this synonym header.
    //          undefined if not a synonym.
    this.synonym = options.synonym;

    // This was added to the tree from an include.
    this.from_include = options.from_include;
    // This was added to the tree from a cirodown_example.
    this.from_cirodown_example = options.from_cirodown_example;

    // Includes under this header.
    this.includes = [];
    this.xss_safe = options.xss_safe;

    // Array of AstNode of synonym headers of this one that have title2 set.
    this.title2s = []

    // Has this node already been validated? This cannot happen twice e.g. because of booleans:
    // booleans currently just work by adding a dummy argument with a default. But then we lose
    // the fact if the thing was given or not. This matters for example for numbered, which
    // inherits from parent if the arg is not given on parent.
    this.validated = false


    this.count_words = options.count_words
    if (this.node_type === AstType.PLAINTEXT) {
      this.word_count = this.text.split(/\s+/).filter(i => i).length;
    } else {
      this.word_count = 0;
    }
    for (const argname in args) {
      const arg = args[argname]
      this.setup_argument(argname, arg);
      for (const ast of arg) {
        ast.parent_node = this
      }
    }
  }

  /**
   * Convert this AST node to an output string.
   *
   * @param {Object} context
   *        If a call will change this object, it must first make a copy,
   *        otherwise future calls to non descendants would also be affected by the change.
   *
   *        - {Object} options - global options passed in from toplevel. Never modified by calls
   *        - {bool} html_is_attr - are we inside an HTML attribute, which implies implies different
   *                 escape rules, e.g. " and ' must be escaped.
   *        - {bool} html_escape - if false, disable HTML escaping entirely. This is needed for
   *                 content that is passed for an external tool for processing, for example
   *                 Math equations to KaTeX, In that case, the arguments need to be passed as is,
   *                 otherwise e.g. `1 < 2` would escape the `<` to `&lt;` and KaTeX would receive bad input.
   *        - {HeaderTreeNode} header_tree - HeaderTreeNode graph containing AstNode headers
   *        - {Object} ids - map of document IDs to their description:
   *                 - 'prefix': prefix to add for a  full reference, e.g. `Figure 1`, `Section 2`, etc.
   *                 - {AstArgument} 'title': the title of the element linked to
   *        - {bool} in_caption_number_visible
   *        - {Set[AstNode]} x_parents: set of all parent x elements.
   *        - {String} root_relpath_shift - relative path introduced due to a scope in split header mode
   * @param {Object} context
   * @return {String}
   */
  render(context) {
    if (context === undefined) {
      context = {};
    }
    if (!('errors' in context)) {
      context.errors = [];
    }
    if (!('html_escape' in context)) {
      context.html_escape = true;
    }
    if (!('html_is_attr' in context)) {
      context.html_is_attr = false;
    }
    if (!('id_provider' in context)) {
      context.id_provider = {};
    }
    if (!('id_conversion' in context)) {
      context.id_conversion = false;
    }
    if (!('ignore_errors' in context)) {
      context.ignore_errors = false;
    }
    if (!('katex_macros' in context)) {
      context.katex_macros = {};
    }
    if (!('macros' in context)) {
      throw new Error('context does not have a mandatory .macros property');
    }
    if (!('source_location' in context)) {
      // Set a source location to this element and the entire subtree.
      // This allows for less verbose manual construction of trees with
      // a single dummy source_location.
      context.source_location = undefined;
    }
    if (!('validate_ast' in context)) {
      // Do validate_ast to this element and the entire subtree.
      // This allows for less verbose manual construction of trees with
      // a single dummy source_location.
      context.validate_ast = false;
    }
    if (!('x_parents' in context)) {
      context.x_parents = new Set();
    }
    if (
      this.from_include &&
      context.in_split_headers &&
      !this.from_cirodown_example
    ) {
      return '';
    }
    if (context.source_location !== undefined) {
      this.source_location = context.source_location;
      for (const argname in this.args) {
        this.args[argname].source_location = context.source_location;
      }
    }
    if (context.validate_ast && !this.validated) {
      validate_ast(this, context);
    }
    if (this.node_type === AstType.PARAGRAPH) {
      // Possible for AstType === PARAGRAPH which can happen for
      // insane paragraph inside header or ID during post processing.
      // Not the nicest solution, but prevents the crash, so so be it.
      // https://github.com/cirosantilli/cirodown/issues/143
      return ' '
    }
    const macro = context.macros[this.macro_name];
    let out;
    if (this.validation_error === undefined) {
      let output_format;
      if (context.id_conversion) {
        output_format = OUTPUT_FORMAT_ID;
      } else {
        output_format = context.options.output_format;
      }
      const convert_function = macro.convert_funcs[output_format];
      if (convert_function === undefined) {
        const message = `output format ${context.options.output_format} not defined for macro ${this.macro_name}`;
        render_error(context, message, this.source_location);
        out = error_message_in_output(message, context);
      } else {
        out = convert_function(this, context);
      }
    } else {
      render_error(
        context,
        this.validation_error[0],
        this.validation_error[1],
        this.validation_error[2]
      );
      out = error_message_in_output(this.validation_error[0], context);
    }

    // Add a div to all direct children of toplevel to implement
    // the on hover links to self and left margin.
    {
      const parent_node = this.parent_node;
      if (
        parent_node !== undefined &&
        parent_node.macro_name === Macro.TOPLEVEL_MACRO_NAME &&
        this.id !== undefined &&
        macro.toplevel_link
      ) {
        out = TOPLEVEL_CHILD_MODIFIER[context.options.output_format](this, context, out);
      }
    }

    return out;
  }

  add_argument(argname, arg) {
    this.args[argname] = arg
    this.setup_argument(argname, arg)
  }

  /* Get parent ID, but only consider IDs that come through header_tree_node. */
  get_local_header_parent_id() {
    if (
      this.header_tree_node !== undefined &&
      this.header_tree_node.parent_node !== undefined &&
      this.header_tree_node.parent_node.ast !== undefined
    ) {
      return this.header_tree_node.parent_node.ast.id
    }
    return undefined
  }

  /** Works with both actual this.header_tree_node and
   * this.header_parent_ids when coming from a database. */
  get_header_parent_ids(context) {
    const ret = new Set()
    const local_parent_id = this.get_local_header_parent_id()
    // Refs defined in the current .ciro file + include_path_set
    if (local_parent_id !== undefined) {
      ret.add(local_parent_id)
    }

    // Refs not defined from outside in the current .ciro file + include_path_set
    // but which were explicitly requested, e.g. we request it for all headers
    // to look for external include parents
    if (context.options.id_provider) {
      const parents_from_db = context.options.id_provider.get_refs_to_as_ids(
        REFS_TABLE_PARENT,
        this.id,
      )
      for (const parent_from_db of parents_from_db) {
        ret.add(parent_from_db)
      }
    }

    // Refs not defined from outside in the current .ciro file + include_path_set,
    // but which were automatically fetched by JOIN our fetch all IDs query:
    // we fetch all parent and children via JOIN for every ID we fetch.
    // This is needed notably for toplevel scope removal.
    for (const header_parent_id of this.header_parent_ids) {
      ret.add(header_parent_id)
    }
    return ret
  }

  /* Like get_header_parent_ids, but returns the parent AST. */
  get_header_parent_asts(context) {
    const ret = []
    const header_parent_ids = this.get_header_parent_ids(context);
    for (const header_parent_id of header_parent_ids) {
      if (header_parent_id !== undefined) {
        ret.push(context.id_provider.get(header_parent_id, context));
      }
    }
    return ret
  }

  is_header_local_descendant_of(ancestor, context) {
    let cur_ast = this;
    const ancestor_id = ancestor.id;
    while (true) {
      let cur_id = cur_ast.get_local_header_parent_id()
      if (cur_id === undefined) {
        return false;
      }
      cur_ast = context.id_provider.get(cur_id, context)
      if (cur_ast.id === ancestor_id) {
        return true;
      }
    }
  }

  /** Manual implementation. There must be a better way, but I can't find it...
   *
   * * row: a row from the Ids database
   */
  static fromJSON(ast_json, context) {
    // Post order depth first convert the AST JSON tree.
    let new_ast
    let ast_head = JSON.parse(ast_json)
    const toplevel_arg = []
    const todo_visit = [[toplevel_arg, ast_head]];
    while (todo_visit.length !== 0) {
      const [parent_arg, ast_json] = todo_visit[todo_visit.length - 1];

      let finishedSubtrees = false
      let done = false
      for (const argname in ast_json.args) {
        const asts = ast_json.args[argname].asts
        if (ast_head === asts[asts.length - 1]) {
          finishedSubtrees = true
          done = true
          break
        }
        if (done) {
          break
        }
      }

      let isLeaf = true
      for (const arg_name in ast_json.args) {
        const arg_json = ast_json.args[arg_name];
        if (arg_json.asts.length > 0) {
          isLeaf = false
          break
        }
      }

      if (finishedSubtrees || isLeaf) {
        todo_visit.pop()

        // Visit.
        const new_args = {}
        for (const arg_name in ast_json.args) {
          const arg_json = ast_json.args[arg_name];
          const new_arg = new AstArgument(arg_json.asts, arg_json.source_location);
          new_args[arg_name] = new_arg
        }
        new_ast = new AstNode(
          AstType[ast_json.node_type],
          ast_json.macro_name,
          new_args,
          ast_json.source_location,
          {
            text: ast_json.text,
            first_toplevel_child: ast_json.first_toplevel_child,
            header_tree_node_word_count: ast_json.header_tree_node_word_count,
            is_first_header_in_input_file: ast_json.is_first_header_in_input_file,
            scope: ast_json.scope,
            split_default: ast_json.split_default,
            synonym: ast_json.synonym,
            word_count: ast_json.word_count,
          }
        );
        if (context !== undefined) {
          validate_ast(new_ast, context)
        }
        parent_arg.push(new_ast)

        ast_head = new_ast
      } else {
        for (const arg_name in ast_json.args) {
          const arg_json = ast_json.args[arg_name];
          for (const ast_child_json of arg_json.asts.slice().reverse()) {
            todo_visit.push([arg_json.asts, ast_child_json]);
          }
          arg_json.asts.length = 0
        }
      }
    }
    return new_ast
  }

  setup_argument(argname, arg) {
    // TODO the second post process pass is destroying this information.
    arg.parent_node = this;
    arg.argument_name = argname;
  }

  // Recursively set source_location on this subtree for elements that
  // don't have it yet. Convenient for manually created trees.
  set_source_location(source_location) {
    const todo_visit_asts = [this]
    const todo_visit_args = []
    while (todo_visit_asts.length > 0) {
      const ast = todo_visit_asts.pop();
      if (ast.source_location === undefined) {
        ast.source_location = source_location
      }
      for (const argname in ast.args) {
        todo_visit_args.push(ast.args[argname])
      }
      while (todo_visit_args.length > 0) {
        const arg = todo_visit_args.pop();
        if (arg.source_location === undefined) {
          arg.source_location = source_location
        }
        for (const ast of arg) {
          todo_visit_asts.push(ast)
        }
      }
    }
  }

  // Set attrs to this AstNode and all its descdendants.
  set_recursively(attrs) {
    const todo_visit_asts = [this]
    const todo_visit_args = []
    while (todo_visit_asts.length > 0) {
      const ast = todo_visit_asts.pop();
      Object.assign(ast, attrs)
      for (const argname in ast.args) {
        todo_visit_args.push(ast.args[argname])
      }
      while (todo_visit_args.length > 0) {
        const arg = todo_visit_args.pop();
        for (const ast of arg) {
          todo_visit_asts.push(ast)
        }
      }
    }
  }

  // A simplified recursive view of the most important fields of this AstNode and children,
  // with one AstNode or AstArgument per line. Indispensable for debugging, since the toJSON
  // is huge.
  toString() {
    let ret = []
    let indent_marker = '  '
    const todo_visit = [{type: 'ast', value: this, indent: 0}]
    while (todo_visit.length > 0) {
      const thing = todo_visit.pop();
      const indent = thing.indent + 1
      const indent_str = indent_marker.repeat(thing.indent)
      if (thing.type === 'ast') {
        const ast = thing.value
        let plaintext
        if (ast.node_type === AstType.PLAINTEXT) {
          plaintext = ` ${JSON.stringify(ast.text)}`
        } else {
          plaintext = ''
        }
        let idstr
        if (ast.id) {
          idstr = ` id="${ast.id}"`
        } else {
          idstr = ''
        }
        const pref = `${indent_str}ast `
        let typestr
        if (ast.node_type === AstType.PLAINTEXT || ast.node_type === AstType.MACRO) {
          typestr = `${ast.macro_name}${plaintext}${idstr}`
        } else {
          typestr = `${ast.node_type.toString()}`
        }
        ret.push(`${pref}${typestr}`)
        if (ast.node_type === AstType.MACRO) {
          const args = ast.args
          const argnames = Object.keys(args).sort().reverse()
          for (const argname of argnames) {
            todo_visit.push({type: 'arg', value: args[argname], argname, indent })
          }
        }
      } else {
        const arg = thing.value
        if (arg) {
          ret.push(`${indent_str}arg ${thing.argname}`)
          for (const ast of arg.slice().reverse()) {
            todo_visit.push({type: 'ast', value: ast, indent })
          }
        }
      }
    }
    return ret.join('\n')
  }

  toJSON() {
    const ret = {
      macro_name: this.macro_name,
      node_type:  symbol_to_string(this.node_type),
      scope:      this.scope,
      source_location: this.source_location,
      text:       this.text,
      first_toplevel_child: this.first_toplevel_child,
      is_first_header_in_input_file: this.is_first_header_in_input_file,
      split_default: this.split_default,
      synonym:    this.synonym,
      word_count: this.word_count,
    }
    const args_given = Object.assign({}, this.args)
    for (const argname in this.args) {
      if (
        argname in this.validation_output &&
        !this.validation_output[argname].given
      ) {
        delete args_given[argname]
      }
    }
    ret.args = args_given
    if (this.header_tree_node !== undefined) {
      ret.header_tree_node_word_count = this.header_tree_node.word_count
    }
    return ret;
  }
}
exports.AstNode = AstNode;

class AstArgument {
  /** @param {List[AstNode]} nodes
   *  @ param {SourceLocation} source_location
   */
  constructor(asts, source_location) {
    if (asts === undefined) {
      this.asts = []
    } else {
      this.asts = asts
    }
    this.source_location = source_location;
    // AstNode
    this.parent_node = undefined;
    // String
    this.argument_name = undefined;
    let i = 0;
    for (const ast of this.asts) {
      ast.parent_argument = this;
      ast.parent_argument_index = i;
      i++;
    };
  }

  concat(...other) {
    return this.asts.concat(...other)
  }

  get(i) {
    return this.asts[i]
  }

  length() {
    return this.asts.length
  }

  map(fn) {
    return this.asts.map(fn)
  }

  set(i, val) {
    this.asts[i] = val
  }

  slice(start, end) {
    return new AstArgument(this.asts.slice(start, end), this.source_location)
  }

  splice(start, deleteCount, ...items) {
    return this.asts.splice(start, deleteCount, ...items)
  }

  reverse() {
    this.asts.reverse()
    return this
  }

  push(...new_asts) {
    const old_length = this.asts.length;
    const ret = this.asts.push(...new_asts);
    let i = 0;
    for (const ast of new_asts) {
      ast.parent_argument = this;
      ast.parent_argument = this;
      ast.parent_argument_index = old_length + i;
      i++;
    }
    return ret;
  }

  reset() {
    this.asts = []
  }

  *[Symbol.iterator] () {
    for (const v of this.asts) {
      yield(v)
    }
  }

  toJSON() {
    return {
      asts: this.asts,
      source_location: this.source_location,
    }
  }
}

class ErrorMessage {
  /**
   * @param {number} severity 1: most severe, 2: next, 3...
   */
  constructor(message, source_location, severity=1) {
    this.message = message;
    this.source_location = source_location;
    this.severity = severity;
  }

  toString(path) {
    let ret = 'error: ';
    let had_line_or_col = false;
    if (this.source_location.path !== undefined) {
      ret += `${this.source_location.path}:`;
    }
    if (this.source_location.line !== undefined) {
      ret += `${this.source_location.line}`;
      had_line_or_col = true;
    }
    if (this.source_location.column !== undefined) {
      if (this.source_location.line !== undefined) {
        ret += `:`;
      }
      ret += `${this.source_location.column}`;
      had_line_or_col = true;
    }
    if (had_line_or_col)
      ret += ': ';
    ret += this.message;
    return ret
  }
}

class FileProvider {
  /* Get entry by path.
   * @return Object
   *        - path {String}: source path
   *        - toplevel_id {String}
   */
  get(path) {
    return this.get_path_entry(path);
  }

  get_path_entry(path) { throw new Error('unimplemented'); }

  async get_path_entry_fetch(path, context) { throw new Error('unimplemented'); }
}
exports.FileProvider = FileProvider;

/** Interface to retrieving the nodes of IDs defined in external files.
 *
 * We need the abstraction because IDs will come from widely different locations
 * between browser and local Node.js operation:
 *
 * - browser: HTTP requests
 * - local: sqlite database
 */
class IdProvider {
  /**
   * @return remove all IDs from this ID provider for the given path.
   *         For example, on a local ID database cache, this would clear
   *         all IDs from the cache.
   */
  clear(input_path_noext_renamed) { throw new Error('unimplemented'); }

  /**
   * @param {String} id
   * @param {String} current_scope: scope node of the location
   *         from which ID get is being done. This affects the final
   *         ID obtained due to scope resolution.
   * @return {Union[AstNode,undefined]}.
   *         undefined: ID not found
   *         Otherwise, the ast node for the given ID
   */
  get(id, context, current_scope) {
    if (id[0] === Macro.HEADER_SCOPE_SEPARATOR) {
      return this.get_noscope(id.substr(1), context);
    } else {
      if (current_scope !== undefined) {
        current_scope += Macro.HEADER_SCOPE_SEPARATOR
        for (let i = current_scope.length - 1; i > 0; i--) {
          if (current_scope[i] === Macro.HEADER_SCOPE_SEPARATOR) {
            let resolved_scope_id = this.get_noscope(
              current_scope.substring(0, i + 1) + id, context);
            if (resolved_scope_id !== undefined) {
              return resolved_scope_id;
            }
          }
        }
      }
      // Not found with ID resolution, so just try to get the exact ID.
      return this.get_noscope(id, context);
    }
  }

  /** Like get, but do not resolve scope. */
  get_noscope(id) {
    return this.get_noscope_base(id);
  }

  get_noscope_raw(ids) { throw new Error('unimplemented'); }

  get_noscope_base(id) {
    return this.get_noscopes_base(new Set([id]))[0]
  }

  get_noscopes_base(ids, ignore_paths_set) { throw new Error('unimplemented'); }

  /** Array[{id: String, defined_at: String}] */
  get_refs_to(type, to_id, reversed=false) { throw new Error('unimplemented'); }

  /**
   * TODO this should be done as single JOIN.
   *
   * @param {String} id
   * @return {Array[AstNode]}: all header nodes that have the given ID
   *                           as a parent includer.
   */
  get_refs_to_as_asts(type, to_id, context, reversed=false) {
    let ref_ids = this.get_refs_to_as_ids(type, to_id, reversed)
    let ret = [];
    for (const ref_id of ref_ids) {
      const from_ast = this.get(ref_id, context);
      if (from_ast === undefined) {
        throw new Error(`could not find reference in database: ${ref_id}`);
      } else {
        ret.push(from_ast);
      }
    }
    return ret;
  }

  /** @return Set[string] the IDs that reference the given AST .*/
  get_refs_to_as_ids(type, to_id, reversed=false) {
    let other_key
    const entries = this.get_refs_to(type, to_id, reversed)
    return new Set(entries.map(e => e.id))
  }
}
exports.IdProvider = IdProvider;

/** IdProvider that first tries id_provider_1 and then id_provider_2.
 *
 * The initial use case for this is to transparently use either IDs defined
 * in the current document, or IDs defined externally.
 */
class ChainedIdProvider extends IdProvider {
  constructor(id_provider_1, id_provider_2) {
    super();
    this.id_provider_1 = id_provider_1;
    this.id_provider_2 = id_provider_2;
  }

  get_noscope_base(id) {
    let ret = this.id_provider_1.get_noscope_base(id);
    if (ret !== undefined) {
      return ret;
    }
    ret = this.id_provider_2.get_noscope_base(id);
    if (ret !== undefined) {
      return ret;
    }
    return undefined;
  }

  get_refs_to(type, to_id, reverse=false) {
    return this.id_provider_1.get_refs_to(type, to_id, reverse).concat(
      this.id_provider_2.get_refs_to(type, to_id, reverse))
  }
}

/** ID provider from a dict.
 * The initial use case is to represent locally defined IDs, and inject
 * them into ChainedIdProvider together with externally defined IDs.
 */
class DictIdProvider extends IdProvider {
  constructor(dict, refs_to) {
    super();
    this.dict = dict;
    this.refs_to = refs_to;
  }

  get_noscope_base(id) {
    return this.dict[id];
  }

  get_refs_to(type, to_id, reverse=false) {
    const ret = []
    const from_ids_reverse = this.refs_to[reverse]
    if (from_ids_reverse !== undefined) {
      const from_ids_type = from_ids_reverse[to_id];
      if (from_ids_type !== undefined) {
        const from_ids = from_ids_type[type];
        if (from_ids !== undefined) {
          for (const from_id in from_ids) {
            const defined_ats = from_ids[from_id].defined_at
            for (const defined_at of defined_ats) {
              ret.push({ id: from_id, defined_at })
            }
          }
        }
      }
    }
    return ret
  }
}

class MacroArgument {
  /**
   * @param {String} name
   */
  constructor(options) {
    if (!('elide_link_only' in options)) {
      // If the only thing contained in this argument is a single
      // Macro.LINK_MACRO_NAME macro, AST post processing instead extracts
      // the href of that macro, and transforms it into a text node with that href.
      //
      // Goal: to allow the use to write both \a[http://example.com] and
      // \p[http://example.com] and get what a sane person expects, see also:
      // https://cirosantilli.com/cirodown#insane-link-parsing-rules
      options.elide_link_only = false;
    }
    if (!('boolean' in options)) {
      // https://cirosantilli.com/cirodown#boolean-named-arguments
      options.boolean = false;
    }
    if (!('count_words' in options)) {
      options.count_words = false;
    }
    if (!('default' in options)) {
      // https://cirosantilli.com/cirodown#boolean-named-arguments
      options.default = undefined;
    }
    if (!('mandatory' in options)) {
      // https://cirosantilli.com/cirodown#mandatory-positional-arguments
      options.mandatory = false;
    }
    if (!('multiple' in options)) {
      // https://cirosantilli.com/cirodown#multiple-argument
      options.multiple = false;
    }
    if (!('positive_nonzero_integer' in options)) {
      options.positive_nonzero_integer = false;
    }
    if (!('remove_whitespace_children' in options)) {
      // https://cirosantilli.com/cirodown#remove_whitespace_children
      options.remove_whitespace_children = false;
    }
    this.boolean = options.boolean;
    this.count_words = options.count_words;
    this.multiple = options.multiple;
    this.default = options.default;
    this.elide_link_only = options.elide_link_only;
    this.mandatory = options.mandatory;
    this.name = options.name;
    this.positive_nonzero_integer = options.positive_nonzero_integer;
    this.remove_whitespace_children = options.remove_whitespace_children;
  }
}

class Macro {
  /**
   * Encapsulates properties of macros, including how to convert
   * them to various output formats.
   *
   * @param {String} name
   * @param {Array[MacroArgument]} args
   * @param {Function} convert
   * @param {Object} options
   *        {boolean} phrasing - is this phrasing content?
   *                  (HTML5 elements that can go in paragraphs). This matters to:
   *                  - determine where `\n\n` paragraphs will split
   *                  - phrasing content does not get IDs
   *        {String} auto_parent - automatically surround consecutive sequences of macros with
   *                 the same parent auto_parent into a node with auto_parent type. E.g.,
   *                 to group list items into ul.
   *        {Set[String]} auto_parent_skip - don't do auto parent generation if the parent is one of these types.
   *        {Function[AstNode, Object] -> String} get_number - return the number that shows on on full references
   *                 as a string, e.g. "123" in "Figure 123." or "1.2.3" in "Section 1.2.3.".
   *                 A return of undefined means that the number is not available, e.g. this is current limitation
   *                 of cross references to other files (could be implemented).
   *        {Function[AstNode, Object] -> Bool} macro_counts_ignore - if true, then an ID should not be automatically given
   *                 to this node. This is usually the case for nodes that are not visible in the final output,
   *                 otherwise that would confuse readers.
   */
  constructor(name, positional_args, html_render_func, options={}) {
    if (!('auto_parent' in options)) {
      // https://cirosantilli.com/cirodown#auto_parent
      options.auto_parent = undefined;
    }
    if (!('auto_parent_skip' in options)) {
      options.auto_parent_skip = new Set();
    }
    if (!('caption_number_visible' in options)) {
      options.caption_number_visible = function(ast) { return false; }
    }
    if (!('caption_prefix' in options)) {
      options.caption_prefix = capitalize_first_letter(name);
    }
    if (!('default_x_style_full' in options)) {
      options.default_x_style_full = true;
    }
    if (!('get_number' in options)) {
      options.get_number = function(ast, context) { return ast.macro_count_visible; }
    }
    if (!('get_title_arg' in options)) {
      options.get_title_arg = function(ast, context) {
        return ast.args[Macro.TITLE_ARGUMENT_NAME];
      }
    }
    if (!('id_prefix' in options)) {
      options.id_prefix = title_to_id(name);
    }
    if (!('image_video_content_func' in options)) {
      options.image_video_content_func = function() { throw new Error('unimplemented'); };
    }
    if (!('macro_counts_ignore' in options)) {
      // Applications:
      // * if an AST node with an ID but no-rendered HTML ID, this breaks editor scroll sync
      //   Returning true makes it not have an ID at all.
      options.macro_counts_ignore = function(ast) {
        return false;
      }
    }
    if (!('named_args' in options)) {
      options.named_args = [];
    }
    if (!('phrasing' in options)) {
      options.phrasing = false;
    }
    if (!('show_disambiguate' in options)) {
      options.show_disambiguate = false;
    }
    if (!('source_func' in options)) {
      options.source_func = function() { throw new Error('unimplemented'); };
    }
    if (!('toplevel_link' in options)) {
      options.toplevel_link = true;
    }
    if (!('xss_safe' in options)) {
      options.xss_safe = true;
    }
    this.name = name;
    this.positional_args = positional_args;
    {
      let named_args = {};
      for (const arg of options.named_args) {
        named_args[arg.name] = arg;
      }
      this.named_args = named_args;
    }
    this.auto_parent = options.auto_parent;
    this.auto_parent_skip = options.auto_parent_skip;
    // This produces incredibly superior render backtraces as you can immediately spot which
    // type of macro is being rendered without entering the line numbers.
    Object.defineProperty(html_render_func, 'name', { value: `html_render_func_${name}`  })
    this.convert_funcs = {
      html: html_render_func
    }
    this.id_prefix = options.id_prefix;
    this.options = options;
    this.remove_whitespace_children = options.remove_whitespace_children;
    this.toplevel_link = options.toplevel_link;
    this.name_to_arg = {};
    for (const arg of this.positional_args) {
      let name = arg.name;
      this.check_name(name);
      this.name_to_arg[name] = arg;
    }
    for (const name in this.named_args) {
      this.check_name(name);
      this.name_to_arg[name] = this.named_args[name];
    }
    // Add arguments common to all macros.
    for (const argname of Macro.COMMON_ARGNAMES) {
      this.named_args[argname] = new MacroArgument({
        name: argname,
      })
      this.name_to_arg[argname] = this.named_args[argname];
    }
    delete this.options.named_args;
  }

  add_convert_function(output_format, my_function) {
    this.convert_funcs[output_format] = my_function;
  }

  check_name(name) {
    if (Macro.COMMON_ARGNAMES.has(name)) {
      throw new Error(`name "${name}" is reserved and automatically added`);
    }
    if (name in this.name_to_arg) {
      throw new Error('name already taken: ' + name);
    }
  }

  toJSON() {
    const options = this.options;
    const ordered_options = {};
    Object.keys(options).sort().forEach(function(key) {
      ordered_options[key] = options[key];
    });
    return {
      name: this.name,
      options: ordered_options,
      positional_args: this.positional_args,
      named_args: this.named_args,
    }
  }
}
exports.Macro = Macro;

// Macro names defined here are those that have magic properties, e.g.
// headers are used by the 'toc'.
Macro.CIRODOWN_EXAMPLE_MACRO_NAME = 'CirodownExample';
Macro.CODE_MACRO_NAME = 'c';
// Add arguments common to all macros.
Macro.DISAMBIGUATE_ARGUMENT_NAME = 'disambiguate';
Macro.ID_ARGUMENT_NAME = 'id';
// Undocumented argument used only for testing.
// Adds data-X to one of the rendered HTML elements.
// to facilitate XPath selection.
Macro.TEST_DATA_ARGUMENT_NAME = 'cirodownTestData';
Macro.TEST_DATA_HTML_PROP = 'data-cirodown-test';
Macro.SYNONYM_ARGUMENT_NAME = 'synonym';
Macro.COMMON_ARGNAMES = new Set([
  Macro.DISAMBIGUATE_ARGUMENT_NAME,
  Macro.ID_ARGUMENT_NAME,
  Macro.TEST_DATA_ARGUMENT_NAME,
]);
Macro.HEADER_MACRO_NAME = 'H';
Macro.HEADER_CHILD_ARGNAME = 'child';
Macro.HEADER_TAG_ARGNAME = 'tag';
Macro.X_MACRO_NAME = 'x';
Macro.HEADER_SCOPE_SEPARATOR = '/';
Macro.INCLUDE_MACRO_NAME = 'Include';
Macro.LINK_MACRO_NAME = 'a';
Macro.LIST_MACRO_NAME = 'L';
Macro.MATH_MACRO_NAME = 'm';
Macro.PARAGRAPH_MACRO_NAME = 'P';
Macro.PLAINTEXT_MACRO_NAME = 'plaintext';
Macro.TABLE_MACRO_NAME = 'Table';
Macro.TD_MACRO_NAME = 'Td';
Macro.TH_MACRO_NAME = 'Th';
Macro.TR_MACRO_NAME = 'Tr';
Macro.TITLE_ARGUMENT_NAME = 'title';
Macro.TITLE2_ARGUMENT_NAME = 'title2';
Macro.TOC_MACRO_NAME = 'Toc';
// We set a fixed magic ID to the ToC because:
// - when doing --split-headers, the easy approach is to add the ToC node
//   after normal ID indexing has happened, which means that we can't link
//   to the ToC as other normal links. And if we could, we would have to worry
//   about how to avoid ID duplication
// - only a single ToC ever renders per document. So we can just have a fixed
//   magic one.
Macro.TOC_ID = 'toc';
Macro.RESERVED_IDS = new Set([
  Macro.TOC_ID,
]);
Macro.TOC_PREFIX = Macro.TOC_ID + '-'
Macro.TOPLEVEL_MACRO_NAME = 'Toplevel';

/** Helper to create plaintext nodes, since so many of the fields are fixed in that case. */
class PlaintextAstNode extends AstNode {
  constructor(text, source_location) {
    super(AstType.PLAINTEXT, Macro.PLAINTEXT_MACRO_NAME,
      {}, source_location, { text });
  }
}

class SourceLocation {
  constructor(line, column, path) {
    this.line = line;
    this.column = column;
    this.path = path;
  }
  clone() {
    // https://stackoverflow.com/questions/41474986/how-to-clone-a-javascript-es6-class-instance
    return lodash.clone(this);
  }
  isEqual(other) {
    return lodash.isEqual(this, other);
  }
}
exports.SourceLocation = SourceLocation;

class Token {
  /**
   * @param {String} type
   * @param {SourceLocation} source_location
   * @param {String} value - Default: undefined
   */
  constructor(type, source_location, value) {
    this.type = type;
    this.source_location = source_location;
    this.value = value;
  }

  toJSON() {
    return {
      type:   this.type.toString(),
      source_location: this.source_location,
      value:  this.value
    }
  }
}

class Tokenizer {
  /**
   * @param {String} input_string
   */
  constructor(
    input_string,
    extra_returns={},
    show_tokenize=false,
    start_line=1,
    input_path=undefined
  ) {
    this.chars = Array.from(input_string);
    this.cur_c = this.chars[0];
    this.source_location = new SourceLocation(start_line, 1, input_path);
    this.extra_returns = extra_returns;
    this.extra_returns.errors = [];
    this.i = 0;
    this.in_insane_header = false;
    this.in_escape_insane_link = false;
    this.list_level = 0;
    this.tokens = [];
    this.show_tokenize = show_tokenize;
    this.log_debug('Tokenizer');
    this.log_debug(`this.chars ${this.chars}`);
    this.log_debug(`this.chars.length ${this.chars.length}`);
    this.log_debug('');
  }

  /** Advance the current character and set cur_c to the next one.
   *
   * Maintain the newline count up to date for debug messages.
   *
   * The current index must only be incremented through this function
   * and never directly.
   *
   * @param {Number} how many to consume
   * @return {boolean} true iff we are not reading past the end of the input
   */
  consume(n=1) {
    for (let done = 0; done < n; done++) {
      this.log_debug('consume');
      this.log_debug('this.i: ' + this.i);
      this.log_debug('this.cur_c: ' + this.cur_c);
      this.log_debug();
      if (this.chars[this.i] === '\n') {
        this.source_location.line += 1;
        this.source_location.column = 1;
      } else {
        this.source_location.column += 1;
      }
      this.i += 1;
      if (this.i >= this.chars.length) {
        this.cur_c = undefined;
        return false;
      }
      this.cur_c = this.chars[this.i];
    }
    return true;
  }

  consume_list_indent() {
    if (this.i > 0 && this.chars[this.i - 1] === '\n') {
      let new_list_level = 0;
      while (
        array_contains_array_at(this.chars, this.i, INSANE_LIST_INDENT) &&
        new_list_level < this.list_level
      ) {
        for (const c in INSANE_LIST_INDENT) {
          this.consume();
        }
        new_list_level += 1;
      }
      for (let i = 0; i < this.list_level - new_list_level; i++) {
        this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
      }
      this.list_level = new_list_level;
    }
  }

  consume_plaintext_char() {
    return this.plaintext_append_or_create(this.cur_c);
  }

  /**
   * @return {boolean} EOF reached?
   */
  consume_optional_newline(literal) {
    this.log_debug('consume_optional_newline');
    this.log_debug();
    if (
      !this.is_end() &&
      this.cur_c === '\n' &&
      (
        literal ||
        // Insane constructs that start with a newline prevent the skip.
        (
          // Pararaph.
          this.peek() !== '\n' &&
          // Insane start.
          this.tokenize_insane_start(this.i + 1) === undefined
        )
      )
    ) {
    this.log_debug();
      return this.consume();
    }
    return true;
  }

  consume_optional_newline_after_argument() {
    if (
      !this.is_end() &&
      this.cur_c === '\n' &&
      !this.in_insane_header
    ) {
      const full_indent = INSANE_LIST_INDENT.repeat(this.list_level);
      if (
        array_contains_array_at(this.chars, this.i + 1, full_indent + START_POSITIONAL_ARGUMENT_CHAR) ||
        array_contains_array_at(this.chars, this.i + 1, full_indent + START_NAMED_ARGUMENT_CHAR)
      ) {
        this.consume(full_indent.length + 1);
      }
    }
  }

  error(message, source_location) {
    let new_source_location;
    if (source_location === undefined) {
      new_source_location = new SourceLocation();
    } else {
      new_source_location = source_location.clone();
    }
    if (new_source_location.path === undefined)
      new_source_location.path = this.source_location.path;
    if (new_source_location.line === undefined)
      new_source_location.line = this.source_location.line;
    if (new_source_location.column === undefined)
      new_source_location.column = this.source_location.column;
    this.extra_returns.errors.push(
      new ErrorMessage(message, new_source_location));
  }

  is_end() {
    return this.i === this.chars.length;
  }

  log_debug(message='') {
    if (this.show_tokenize) {
      console.error('tokenize: ' + message);
    }
  }

  peek() {
    return this.chars[this.i + 1];
  }

  plaintext_append_or_create(s) {
    let new_plaintext = true;
    if (this.tokens.length > 0) {
      let last_token = this.tokens[this.tokens.length - 1];
      if (last_token.type === TokenType.PLAINTEXT) {
        last_token.value += s;
        new_plaintext = false;
      }
    }
    if (new_plaintext) {
      this.push_token(TokenType.PLAINTEXT, s);
    }
    return this.consume();
  }

  push_token(token, value, source_location) {
    this.log_debug('push_token');
    this.log_debug('token: ' + token.toString());
    this.log_debug('value: ' + value);
    this.log_debug();
    let new_source_location;
    if (source_location === undefined) {
      new_source_location = new SourceLocation();
    } else {
      new_source_location = source_location.clone();
    }
    if (new_source_location.line === undefined)
      new_source_location.line = this.source_location.line;
    if (new_source_location.column === undefined)
      new_source_location.column = this.source_location.column;
    if (new_source_location.path === undefined)
      new_source_location.path = this.source_location.path;
    this.tokens.push(new Token(token, new_source_location, value));
  }

  /**
   * @return {Array[Token]}
   */
  tokenize() {
    // Ignore the last newline of the file.
    // It is good practice to always have a newline
    // at the end of files, but it doesn't really mean
    // that the user wants the last element to contain one.
    if (this.chars[this.chars.length - 1] === '\n') {
      this.chars.pop();
    }
    let unterminated_literal = false;
    let start_source_location;
    while (!this.is_end()) {
      this.log_debug('tokenize loop');
      this.log_debug('this.i: ' + this.i);
      this.log_debug('this.source_location: ' + this.source_location);
      this.log_debug('this.cur_c: ' + this.cur_c);
      if (this.in_insane_header && this.cur_c === '\n') {
        this.in_insane_header = false;
        this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
        this.consume_optional_newline_after_argument()
      }
      this.consume_list_indent();
      start_source_location = this.source_location.clone();
      if (this.cur_c === ESCAPE_CHAR) {
        this.consume();
        if (this.is_end()) {
          // Maybe this should be an error.
        } else if (ESCAPABLE_CHARS.has(this.cur_c)) {
          this.consume_plaintext_char();
        } else {
          // Insane link.
          for (const known_url_protocol of KNOWN_URL_PROTOCOLS) {
            if (array_contains_array_at(this.chars, this.i, known_url_protocol)) {
              this.in_escape_insane_link = true;
              break;
            }
          }
          if (!this.in_escape_insane_link) {
            let macro_name = this.tokenize_func(char_is_identifier);
            this.consume_optional_newline();
            this.push_token(
              TokenType.MACRO_NAME,
              macro_name,
              start_source_location,
            );
          }
        }
      } else if (this.cur_c === START_NAMED_ARGUMENT_CHAR) {
        let source_location = this.source_location.clone();
        // Tokenize past the last open char.
        let open_length = this.tokenize_func(
          c => c === START_NAMED_ARGUMENT_CHAR
        ).length;
        this.push_token(TokenType.NAMED_ARGUMENT_START,
          START_NAMED_ARGUMENT_CHAR.repeat(open_length), source_location);
        source_location = this.source_location.clone();
        let arg_name = this.tokenize_func(char_is_identifier);
        this.push_token(TokenType.NAMED_ARGUMENT_NAME, arg_name, source_location);
        if (this.cur_c === NAMED_ARGUMENT_EQUAL_CHAR) {
          // Consume the = sign.
          this.consume();
        } else if (this.cur_c === END_NAMED_ARGUMENT_CHAR) {
          // Boolean argument.
        } else {
          this.error(`expected character: '${NAMED_ARGUMENT_EQUAL_CHAR}' or '${END_NAMED_ARGUMENT_CHAR}', got '${this.cur_c}'`);
        }
        if (open_length === 1) {
          this.consume_optional_newline();
        } else {
          // Literal argument.
          let close_string = closing_char(
            START_NAMED_ARGUMENT_CHAR).repeat(open_length);
          if (!this.tokenize_literal(START_NAMED_ARGUMENT_CHAR, close_string)) {
            unterminated_literal = true;
          }
          this.push_token(TokenType.NAMED_ARGUMENT_END, close_string);
          this.consume_optional_newline_after_argument()
        }
      } else if (this.cur_c === END_NAMED_ARGUMENT_CHAR) {
        this.push_token(TokenType.NAMED_ARGUMENT_END, END_NAMED_ARGUMENT_CHAR);
        this.consume();
        this.consume_optional_newline_after_argument()
      } else if (this.cur_c === START_POSITIONAL_ARGUMENT_CHAR) {
        let source_location = this.source_location.clone();
        // Tokenize past the last open char.
        let open_length = this.tokenize_func(
          c => c === START_POSITIONAL_ARGUMENT_CHAR
        ).length;
        this.push_token(TokenType.POSITIONAL_ARGUMENT_START,
          START_POSITIONAL_ARGUMENT_CHAR.repeat(open_length), source_location);
        if (open_length === 1) {
          this.consume_optional_newline();
        } else {
          // Literal argument.
          let close_string = closing_char(
            START_POSITIONAL_ARGUMENT_CHAR).repeat(open_length);
          if (!this.tokenize_literal(START_POSITIONAL_ARGUMENT_CHAR, close_string)) {
            unterminated_literal = true;
          }
          this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
          this.consume_optional_newline_after_argument()
        }
      } else if (this.cur_c === END_POSITIONAL_ARGUMENT_CHAR) {
        this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
        this.consume();
        this.consume_optional_newline_after_argument();
      } else if (this.cur_c in MAGIC_CHAR_ARGS) {
        // Insane shortcuts e.g. $$ math and `` code.
        let open_char = this.cur_c;
        let open_length = this.tokenize_func(c => c === open_char).length;
        let close_string = open_char.repeat(open_length);
        let macro_name = MAGIC_CHAR_ARGS[open_char];
        if (open_length > 1) {
          macro_name = macro_name.toUpperCase();
        }
        this.push_token(TokenType.MACRO_NAME, macro_name);
        this.push_token(TokenType.POSITIONAL_ARGUMENT_START);
        if (!this.tokenize_literal(open_char, close_string, true)) {
          unterminated_literal = true;
        }
        this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
        this.consume_optional_newline_after_argument()
      } else if (this.cur_c === '\n' && this.peek() === '\n') {
        this.consume();
        this.consume();
        // We must close list level changes before the paragraph, e.g. in:
        //
        // ``
        // * aa
        // * bb
        //
        // cc
        // ``
        //
        // the paragraph goes after `ul`, it does not stick to `bb`
        this.consume_list_indent();
        this.push_token(TokenType.PARAGRAPH);
        if (this.cur_c === '\n') {
          this.error('paragraph with more than two newlines, use just two');
        }
      } else {
        let done = false;

        // Insane link.
        if (this.in_escape_insane_link) {
          this.in_escape_insane_link = false;
        } else {
          let is_insane_link = false;
          for (const known_url_protocol of KNOWN_URL_PROTOCOLS) {
            if (
              array_contains_array_at(this.chars, this.i, known_url_protocol)
            ) {
              const pos_char_after = this.i + known_url_protocol.length;
              if (
                pos_char_after < this.chars.length &&
                !INSANE_LINK_END_CHARS.has(this.chars[pos_char_after])
              ) {
                is_insane_link = true;
                break;
              }
            }
          }
          if (is_insane_link) {
            this.push_token(TokenType.MACRO_NAME, Macro.LINK_MACRO_NAME);
            this.push_token(TokenType.POSITIONAL_ARGUMENT_START);
            let link_text = '';
            while (this.consume_plaintext_char()) {
              if (INSANE_LINK_END_CHARS.has(this.cur_c)) {
                break;
              }
              if (this.cur_c === ESCAPE_CHAR) {
                this.consume();
              }
            }
            this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
            this.consume_optional_newline_after_argument();
            done = true;
          }
        }

        // Insane lists and tables.
        if (
          !done && (
            this.i === 0 ||
            this.cur_c === '\n' ||
            (
              this.tokens.length > 0 &&
              this.tokens[this.tokens.length - 1].type === TokenType.PARAGRAPH
            ) ||
            // Immediately at the start of an argument.
            this.tokens.length > 0 && (
              this.tokens[this.tokens.length - 1].type === TokenType.NAMED_ARGUMENT_START ||
              this.tokens[this.tokens.length - 1].type === TokenType.POSITIONAL_ARGUMENT_START
            )
          )
        ) {
          let i = this.i;
          if (this.cur_c === '\n') {
            i += 1;
          }
          let new_list_level = 0;
          while (array_contains_array_at(this.chars, i, INSANE_LIST_INDENT)) {
            i += INSANE_LIST_INDENT.length;
            new_list_level += 1;
          }
          let insane_start_return = this.tokenize_insane_start(i);
          if (insane_start_return !== undefined) {
            const [insane_start, insane_start_length] = insane_start_return;
            if (new_list_level <= this.list_level + 1) {
              if (this.cur_c === '\n') {
                this.consume();
              }
              this.consume_list_indent();
              this.push_token(TokenType.MACRO_NAME, INSANE_STARTS_TO_MACRO_NAME[insane_start]);
              this.push_token(TokenType.POSITIONAL_ARGUMENT_START);
              this.list_level += 1;
              done = true;
              for (let i = 0; i < insane_start_length; i++) {
                this.consume();
              }
            }
          }
        }

        // Insane headers.
        if (!done && (
          this.i === 0 ||
          this.chars[this.i - 1] === '\n'
        )) {
          let i = this.i;
          let new_header_level = 0;
          while (this.chars[i] === INSANE_HEADER_CHAR) {
            i += 1;
            new_header_level += 1;
          }
          if (new_header_level > 0 && this.chars[i] === ' ') {
            this.push_token(TokenType.MACRO_NAME, Macro.HEADER_MACRO_NAME);
            this.push_token(TokenType.POSITIONAL_ARGUMENT_START, INSANE_HEADER_CHAR.repeat(new_header_level));
            this.push_token(TokenType.PLAINTEXT, new_header_level.toString());
            this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
            this.push_token(TokenType.POSITIONAL_ARGUMENT_START);
            for (let i = 0; i <= new_header_level; i++)
              this.consume();
            this.in_insane_header = true;
            done = true;
          }
        }

        // Character is nothing else, so finally it is a regular plaintext character.
        if (!done) {
          this.consume_plaintext_char();
        }
      }
    }
    if (unterminated_literal) {
      this.error(`unterminated literal argument`, start_source_location);
    }

    // Close any open headers at the end of the document.
    if (this.in_insane_header) {
      this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
    }

    // Close any open list levels at the end of the document.
    for (let i = 0; i < this.list_level; i++) {
      this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
    }

    this.push_token(TokenType.PARAGRAPH);
    this.push_token(TokenType.INPUT_END);
    return this.tokens;
  }

  // Create a token with all consecutive chars that are accepted
  // by the given function.
  tokenize_func(f) {
    this.log_debug('tokenize_func');
    this.log_debug('this.i: ' + this.i);
    this.log_debug('this.cur_c: ' + this.cur_c);
    this.log_debug('');
    let value = '';
    while (f(this.cur_c)) {
      value += this.cur_c;
      this.consume();
      if (this.is_end())
        break;
    }
    return value;
  }

  /**
   * Determine if we are at the start of an insane indented sequence
   * like an insane list '* ' or table '| '
   *
   * @return {Union[[String,Number],undefined]} -
   *         - [insane_start, length] if any is found. For an empty table or list without space,
   *           length is insane_start.length - 1. Otherwise it equals insane_start.length.
   *         - undefined if none found.
   */
  tokenize_insane_start(i) {
    for (const insane_start in INSANE_STARTS_TO_MACRO_NAME) {
      if (
        array_contains_array_at(this.chars, i, insane_start)
      ) {
        // Full insane start match.
        return [insane_start, insane_start.length];
      }
      // Empty table or list without space.
      let insane_start_nospace = insane_start.substring(0, insane_start.length - 1);
      if (
        array_contains_array_at(this.chars, i, insane_start_nospace) &&
        (
          i === this.chars.length - 1 ||
          this.chars[i + insane_start.length - 1] === '\n'
        )
      ) {
        return [insane_start, insane_start.length - 1];
      }
    }
    return undefined;
  }

  /**
   * Start inside the literal argument after the opening,
   * and consume until its end.
   *
   * @return {boolean} - true if OK, false if unexpected EOF
   */
  tokenize_literal(open_char, close_string, openEqualsClose = false) {
    this.log_debug('tokenize_literal');
    this.log_debug(`this.i: ${this.i}`);
    this.log_debug(`open_char: ${open_char}`);
    this.log_debug(`close_string ${close_string}`);
    this.log_debug('');

    if (this.is_end())
      return false;

    // Remove leading escapes.
    let i = this.i;
    while (this.chars[i] === ESCAPE_CHAR) {
      i++;
      if (this.is_end())
        return false;
    }
    if (this.chars[i] === open_char && !openEqualsClose) {
      // Skip one of the escape chars if they are followed by an open.
      if (!this.consume())
        return false;
    } else {
      if (!this.consume_optional_newline(true))
        return false;
    }

    // Now consume the following unescaped part.
    let start_i = this.i;
    let start_source_location = this.source_location.clone();
    while (
      this.chars.slice(this.i, this.i + close_string.length).join('')
      !== close_string
    ) {
      if (!this.consume())
        return false;
    }
    // Handle trailing escape.
    let append;
    let end_i;
    if (
      this.chars[this.i - 1] === ESCAPE_CHAR &&
      this.chars.slice(this.i + 1, this.i + close_string.length + 1).join('') === close_string
    ) {
      // Ignore the trailing backslash.
      end_i = this.i - 1;
      // Consume the escaped closing char.
      if (!this.consume())
        return false;
      append = closing_char(open_char);
    } else {
      end_i = this.i;
      append = '';
    }

    // Remove insane list indents.
    let plaintext = '';
    {
      let i = start_i;
      while (true) {
        if (this.chars[i - 1] === '\n') {
          if (this.chars[i] === '\n') {
          } else if (array_contains_array_at(this.chars, i, INSANE_LIST_INDENT.repeat(this.list_level))) {
            i += INSANE_LIST_INDENT.length * this.list_level;
          } else {
            this.error(`literal argument with indent smaller than current insane list`, start_source_location);
          }
        }
        if (i < end_i) {
          plaintext += this.chars[i];
        } else {
          break;
        }
        i++;
      }
    }

    // Create the token.
    this.push_token(
      TokenType.PLAINTEXT,
      plaintext + append,
      start_source_location,
    );

    // Skip over the closing string.
    for (let i = 0; i < close_string.length; i++)
      this.consume();
    return true;
  }
}

class HeaderTreeNode {
  /**
   * Structure:
   *
   * toplevel -> value -> toplevel header ast
   *   -> child[0] -> value -> h2 1 header ast
   *               -> parent_node -> toplevel
   *   -> child[1] -> value -> h2 2 header ast
   *               -> parent_node -> toplevel
   *
   * P inside  h2 1:
   *   -> parent_node -> child[0]
   * a inside P inside  h2 1:
   *   -> parent_node -> child[0]
   *
   * And every non-header element also gets a parent link to its header without child down:
   *
   * @param {AstNode} value
   * @param {HeaderTreeNode} parent_node
   */
  constructor(ast, parent_node, options={}) {
    this.ast = ast;
    this.parent_node = parent_node;
    this.children = [];
    this.index = undefined;
    this.descendant_count = 0;
    this.descendant_word_count = 0;
    if (ast !== undefined) {
      this.word_count = ast.word_count;
      if (!ast.in_header) {
        let cur_node = this.parent_node;
        if (cur_node !== undefined && cur_node.parent_node !== undefined) {
          cur_node.update_ancestor_counts(this.word_count + ast.header_tree_node_word_count)
        }
      }
    } else {
      this.word_count = 0;
    }
  }

  add_child(child) {
    child.index = this.children.length;
    this.children.push(child);
    let cur_node = this;
    while (cur_node !== undefined) {
      cur_node.descendant_count += 1;
      cur_node = cur_node.parent_node;
    }
  }

  /** @return {Number} How deep this node is relative to
   * the to of the root of the tree. */
  get_level() {
    let level = 0;
    let cur_node = this.parent_node;
    while (cur_node !== undefined) {
      level++;
      cur_node = cur_node.parent_node;
    }
    return level;
  }

  /** E.g. get number 1.4.2.5 of a Section.
   *
   * @return {String}
   */
  get_nested_number(header_tree_top_level) {
    let indexes = [];
    let cur_node = this;
    while (
      // Possible in skipped header levels.
      cur_node !== undefined &&
      cur_node.ast !== undefined &&
      cur_node.ast.numbered &&
      cur_node.get_level() !== header_tree_top_level
    ) {
      indexes.push(cur_node.index + 1);
      cur_node = cur_node.parent_node;
    }
    return indexes.reverse().join('.');
  }

  toString() {
    const ret = [];
    let todo_visit;
    // False for toplevel of the tree.
    if (this.ast === undefined) {
      todo_visit = this.children.slice();
    } else {
      todo_visit = [this];
    }
    while (todo_visit.length > 0) {
      const cur_node = todo_visit.pop();
      const level = cur_node.get_level();
      ret.push(`${INSANE_HEADER_CHAR.repeat(level)} h${level} ${cur_node.get_nested_number(1)} ${cur_node.ast.id}`);
      todo_visit.push(...cur_node.children.slice().reverse());
    }
    return ret.join('\n');
  }

  update_ancestor_counts(add) {
    let cur_node = this.parent_node
    while (cur_node !== undefined) {
      cur_node.descendant_word_count += add;
      cur_node = cur_node.parent_node;
    }
  }
}
exports.HeaderTreeNode = HeaderTreeNode

/** Add an entry to the data structures that keep the map of incoming
 * and outgoing \x and \x {child} links. */
function add_to_refs_to(toid, context, fromid, relation_type, child_index) {
  add_to_refs_to_one_way(false, toid,   context, fromid, relation_type, child_index)
  add_to_refs_to_one_way(true,  fromid, context, toid,   relation_type, child_index)
}

function add_to_refs_to_one_way(reverse, toid, context, fromid, relation_type, child_index) {
  let from_to_dict_false = context.refs_to[reverse];
  let from_ids;
  if (toid in from_to_dict_false) {
    from_ids = from_to_dict_false[toid];
  } else {
    from_ids = {};
    from_to_dict_false[toid] = from_ids;
  }
  let from_ids_relation_type;
  if (relation_type in from_ids) {
    from_ids_relation_type = from_ids[relation_type]
  } else {
    from_ids_relation_type = {}
    from_ids[relation_type] = from_ids_relation_type
  }
  let from_ids_relation_type_fromid = from_ids_relation_type[fromid]
  if (from_ids_relation_type_fromid === undefined) {
    from_ids_relation_type_fromid = { defined_at: new Set(), child_index }
    from_ids_relation_type[fromid] = from_ids_relation_type_fromid
  }
  from_ids_relation_type_fromid.defined_at.add(
    context.options.input_path
  )
}

/**
 * Determine if big_array contains small_array starting at index position
 * inside the big array.
 *
 * @return {boolean} true iff if the big array contains the small one
 */
function array_contains_array_at(big_array, position, small_array) {
  for (let i = 0; i < small_array.length; i++) {
    if (big_array[position + i] !== small_array[i]) {
      return false;
    }
  }
  return true;
}

// https://stackoverflow.com/questions/7837456/how-to-compare-arrays-in-javascript
function array_equals(arr1, arr2) {
  if (arr1.length !== arr2.length)
    return false;
  for (let i = 0; i < arr1.length; i++) {
    if (arr1[i] !== arr2[i])
      return false;
  }
  return true;
}

function basename(str, sep) {
  return path_split(str, sep)[1];
}

/// https://stackoverflow.com/questions/22697936/binary-search-in-javascript/29018745#29018745
function binary_search(ar, el, compare_fn) {
  var m = 0;
  var n = ar.length - 1;
  while (m <= n) {
    var k = (n + m) >> 1;
    var cmp = compare_fn(el, ar[k]);
    if (cmp > 0) {
      m = k + 1;
    } else if (cmp < 0) {
      n = k - 1;
    } else {
      return k;
    }
  }
  return -m - 1;
}

function binary_search_insert(ar, el, compare_fn) {
  let index = binary_search(ar, el, compare_fn);
  if (index < 0) {
    ar.splice(-(index + 1), 0, el);
  }
  return ar;
}

function binary_search_line_to_id_array_fn(elem0, elem1) {
  return elem0[0] - elem1[0];
}

/** Calculate node ID and add it to the ID index. */
function calculate_id(ast, context, non_indexed_ids, indexed_ids,
  macro_counts, macro_count_global, macro_counts_visible, state, is_header, line_to_id_array
) {
  let title_text
  const macro_name = ast.macro_name;
  if (macro_name === Macro.TOC_MACRO_NAME) {
    ast.id = Macro.TOC_ID;
  } else {
    const macro = context.macros[macro_name];

    // Linear count of each macro type for macros that have IDs.
    if (!macro.options.macro_counts_ignore(ast)) {
      if (!(macro_name in macro_counts)) {
        macro_counts[macro_name] = 0;
      }
      const macro_count = macro_counts[macro_name] + 1;
      macro_counts[macro_name] = macro_count;
      ast.macro_count = macro_count;
    }

    let index_id = true;
    if (
      // This can happen be false for included headers, and this is notably important
      // for the toplevel header which gets its ID from the filename.
      ast.id === undefined
    ) {
      const macro_id_arg = ast.args[Macro.ID_ARGUMENT_NAME];
      if (macro_id_arg === undefined) {
        let id_text = '';
        const id_prefix = context.macros[ast.macro_name].id_prefix;
        const title_arg = macro.options.get_title_arg(ast, context);
        if (title_arg !== undefined) {
          if (id_prefix !== '') {
            id_text += id_prefix + ID_SEPARATOR
          }
          const new_context = clone_and_set(context, 'id_conversion', true);
          new_context.id_conversion_for_header = is_header;
          title_text = render_arg_noescape(title_arg, new_context)
          if (
            ast.macro_name === Macro.HEADER_MACRO_NAME &&
            ast.validation_output.file.given
          ) {
            let id_text_append
            const file_render = render_arg(ast.args.file, new_context)
            if (file_render) {
              id_text_append = file_render
            } else {
              id_text_append = title_text
            }
            ast.file = id_text_append
            id_text = 'file' + Macro.HEADER_SCOPE_SEPARATOR + id_text + id_text_append
          } else {
            id_text += title_to_id(title_text, new_context.options.cirodown_json['id']);
          }
          const disambiguate_arg = ast.args[Macro.DISAMBIGUATE_ARGUMENT_NAME];
          if (disambiguate_arg !== undefined) {
            id_text += ID_SEPARATOR + title_to_id(render_arg_noescape(disambiguate_arg, new_context), new_context.options.cirodown_json['id']);
          }
          ast.id = id_text;
        } else {
          id_text += '_'
        }

        if (ast.id === undefined && !macro.options.phrasing) {
          id_text += macro_count_global;
          macro_count_global++
          ast.id = id_text;
          index_id = false;
          // IDs of type p-1, p-2, q-1, q-2, etc.
          //if (ast.macro_count !== undefined) {
          //  id_text += ast.macro_count;
          //  ast.id = id_text;
          //}
        }
      } else {
        ast.id = render_arg_noescape(macro_id_arg, context);
      }
      if (Macro.RESERVED_IDS.has(ast.id)) {
        let message = `reserved ID "${ast.id}"`;
        parse_error(state, message, ast.source_location);
      }
      if (ast.id !== undefined && ast.scope !== undefined) {
        ast.id = ast.scope + Macro.HEADER_SCOPE_SEPARATOR + ast.id
      }
    }
    ast.index_id = index_id;
    if (ast.id !== undefined && !ast.force_no_index) {
      let non_indexed_ast = non_indexed_ids[ast.id];
      if (non_indexed_ast === undefined) {
        non_indexed_ids[ast.id] = ast;
        if (index_id) {
          indexed_ids[ast.id] = ast;
          const local_id = ast.get_local_header_parent_id()
          if (local_id !== undefined) {
            add_to_refs_to(
              ast.id,
              context,
              local_id,
              REFS_TABLE_PARENT,
              ast.header_tree_node.index,
            );
          }
        }
      } else {
        const message = duplicate_id_error_message(
          ast.id,
          non_indexed_ast.source_location.path,
          non_indexed_ast.source_location.line,
          non_indexed_ast.source_location.column
        )
        parse_error(state, message, ast.source_location);
      }
      if (caption_number_visible(ast, context)) {
        if (!(macro_name in macro_counts_visible)) {
          macro_counts_visible[macro_name] = 0;
        }
        const macro_count = macro_counts_visible[macro_name] + 1;
        macro_counts_visible[macro_name] = macro_count;
        ast.macro_count_visible = macro_count;
      }
      binary_search_insert(line_to_id_array,
        [ast.source_location.line, ast.id], binary_search_line_to_id_array_fn);
    }
  }
  return { title_text, macro_count_global }
}

// Return the full scope of a given node. This includes the concatenation of both:
// * any scopes of any parents
// * the ID of the node if it has a scope set for itself
// If none of those provide scopes, return undefined.
function calculate_scope(ast) {
  let parent_scope;
  if (ast.scope !== undefined) {
    parent_scope = ast.scope;
  }

  let self_scope;
  if (
      ast.validation_output.scope !== undefined &&
      ast.validation_output.scope.boolean
  ) {
    self_scope = ast.id;
    if (parent_scope !== undefined) {
      self_scope = self_scope.substr(parent_scope.length + 1);
    }
  } else {
    self_scope = '';
  }

  let ret = '';
  if (parent_scope !== undefined) {
    ret += parent_scope;
  }
  if (
    parent_scope !== undefined &&
    self_scope !== ''
  ) {
    ret += Macro.HEADER_SCOPE_SEPARATOR;
  }
  if (self_scope !== '') {
    ret += self_scope;
  }
  if (ret === '') {
    return undefined;
  }
  return ret;
}

/* Calculate the length of the scope of a child header given its parent ast. */
function calculate_scope_length(parent_ast) {
  if (parent_ast !== undefined) {
    let scope = calculate_scope(parent_ast);
    if (scope !== undefined) {
      return scope.length + 1;
    }
  }
  return 0;
}

function capitalize_first_letter(string) {
  return string.charAt(0).toUpperCase() + string.slice(1);
}
exports.capitalize_first_letter = capitalize_first_letter

function caption_number_visible(ast, context) {
  return ast.index_id || context.macros[ast.macro_name].options.caption_number_visible(ast, context);
}

function char_is_alphanumeric(c) {
  let code = c.codePointAt(0);
  return (
    // 0-9
    (code > 47 && code < 58) ||
    // A-Z
    (code > 64 && code < 91) ||
    // a-z
    (code > 96 && code < 123)
  )
}

// Valid macro name / argument characters.
// Compatible with JavaScript-like function names / variables.
function char_is_identifier(c) {
  return char_is_alphanumeric(c) || c === '_';
};

/** Shallow clone an object, and set a given value on the cloned one. */
function clone_and_set(obj, key, value) {
  let new_obj = {...obj};
  new_obj[key] = value;
  return new_obj;
}
exports.clone_and_set = clone_and_set;

function clone_object(obj) {
  // https://stackoverflow.com/questions/41474986/how-to-clone-a-javascript-es6-class-instance
  return Object.assign(Object.create(Object.getPrototypeOf(obj)), obj);
}

function closing_char(c) {
  if (c === START_POSITIONAL_ARGUMENT_CHAR)
    return END_POSITIONAL_ARGUMENT_CHAR;
  if (c === START_NAMED_ARGUMENT_CHAR)
    return END_NAMED_ARGUMENT_CHAR;
  throw new Error('char does not have a close: ' + c);
}

function closing_token(token) {
  if (token === TokenType.POSITIONAL_ARGUMENT_START)
    return TokenType.POSITIONAL_ARGUMENT_END;
  if (token === TokenType.NAMED_ARGUMENT_START)
    return TokenType.NAMED_ARGUMENT_END;
  throw new Error('token does not have a close: ' + token);
}

/**
 * Main cirodown input to HTML/LaTeX/etc. output JavaScript API.
 *
 * The CLI interface basically just feeds this.
 *
 * @param {Object} options
 *          {IdProvider} external_ids
 *          {Function[String] -> [string,string]} read_include(id) -> [file_name, content]
 *          {Number} h_include_level_offset - add this offset to the levels of every header
 *          {boolean} render - if false, parse the input, but don't render it,
 *              and return undefined.
 *              The initial use case for this is to allow a faster and error-less
 *              first pass when building an entire directory with internal cross file
 *              references to extract IDs of each file.
 * @param {Object} extra_returns
 *          {Object} rendered_outputs pairs of output_path -> rendered_output_string
 *                   This is needed primarily because --html-spit-header
 *                   generates multiple output files for a single input file.
 * @return {String}
 */
async function convert(
  input_string,
  options,
  extra_returns={},
) {
  let context = convert_init_context(options, extra_returns);

  // Tokenize.
  let sub_extra_returns;
  sub_extra_returns = {};
  perf_print(context, 'tokenize_pre')
  let tokens = (new Tokenizer(
    input_string,
    sub_extra_returns,
    context.options.log.tokenize,
    context.options.start_line,
    context.options.input_path,
  )).tokenize();
  perf_print(context, 'tokenize_post')
  if (context.options.log['tokens-inside']) {
    console.error('tokens:');
    for (let i = 0; i < tokens.length; i++) {
      console.error(`${i}: ${JSON.stringify(tokens[i], null, 2)}`);
    }
    console.error();
  }
  extra_returns.tokens = tokens;
  extra_returns.errors.push(...sub_extra_returns.errors);
  sub_extra_returns = {};

  // Setup context.media_provider_default based on `default-for`.
  {
    const media_providers = context.options.cirodown_json['media-providers'];
    context.media_provider_default = {};
    for (const media_provider_name in media_providers) {
      const media_provider = media_providers[media_provider_name];
      if ('default-for' in media_provider) {
        for (const default_for of media_provider['default-for']) {
          if (default_for[0] == default_for[0].toUpperCase()) {
            context.errors.push(new ErrorMessage(`default-for names must start with a lower case letter`, new SourceLocation(1, 1)));
          } else {
            if (default_for === 'all') {
              for (const macro_name of MACRO_WITH_MEDIA_PROVIDER) {
                context.media_provider_default[default_for] = media_provider_name;
                context.media_provider_default[capitalize_first_letter(default_for)] = media_provider_name;
              }
            } else {
              if (MACRO_WITH_MEDIA_PROVIDER.has(default_for)) {
                if (context.media_provider_default[default_for] === undefined) {
                  context.media_provider_default[default_for] = media_provider_name;
                  context.media_provider_default[capitalize_first_letter(default_for)] = media_provider_name;
                } else {
                  context.errors.push(new ErrorMessage(`multiple media providers set for macro "${default_for}"`, new SourceLocation(1, 1)));
                }
              } else {
                context.errors.push(new ErrorMessage(`macro "${default_for}" does not accept media providers`, new SourceLocation(1, 1)));
              }
            }
          }
        }
      }
    }
    for (const macro_name of MACRO_WITH_MEDIA_PROVIDER) {
      if (context.media_provider_default[macro_name] === undefined) {
        context.media_provider_default[macro_name] = 'local';
        context.media_provider_default[capitalize_first_letter(macro_name)] = 'local';
      }
    }
  }

  let ast = await parse(tokens, context.options, context, sub_extra_returns);
  if (context.options.log['ast-inside']) {
    console.error('ast:');
    console.error(JSON.stringify(ast, null, 2));
    console.error();
  }
  extra_returns.ast = ast;
  extra_returns.context = context;
  extra_returns.ids = sub_extra_returns.ids;
  Object.assign(extra_returns.debug_perf, sub_extra_returns.debug_perf);
  extra_returns.errors.push(...sub_extra_returns.errors);
  let output;
  if (context.options.render) {
    if (context.options.log['split-headers']) {
      console.error('split-headers non-split: ' + context.options.input_path);
    }

    // For non-split header toplevel conversion.
    if (context.options.outfile === undefined) {
      let outpath;
      if (context.options.input_path !== undefined) {
        let id;
        if (context.toplevel_id === undefined) {
          const toplevel_header_node = context.header_tree.children[0];
          if (toplevel_header_node === undefined) {
            id = undefined
          } else {
            const toplevel_header_ast = toplevel_header_node.ast;
            id = toplevel_header_ast.id;
          }
          context.toplevel_id = id;
        } else {
          id = context.toplevel_id;
        }
        if (id !== undefined) {
          outpath = (output_path(
            context.options.input_path,
            id,
            clone_and_set(context, 'to_split_headers', false)
          ))[0];
        }
        if (outpath !== undefined) {
          context.options.template_vars.root_relpath = get_root_relpath(outpath, context);
        }
      }
      context.toplevel_output_path = outpath;
    }

    // First render.
    context.extra_returns.rendered_outputs = {};
    perf_print(context, 'render_pre')
    output = ast.render(context);
    context.katex_macros = Object.assign({}, context.options.katex_macros);
    let toplevel_output_path
    if (context.toplevel_output_path === undefined) {
      toplevel_output_path = INDEX_BASENAME_NOEXT + '.' + HTML_EXT
    } else {
      toplevel_output_path = context.toplevel_output_path
    }
    context.extra_returns.rendered_outputs[toplevel_output_path] = output;

    perf_print(context, 'split_render_pre')
    // Split header conversion.
    if (context.options.split_headers) {
      const content = ast.args.content;
      // Gather up each header (must be a direct child of toplevel)
      // and all elements that belong to that header (up to the next header).
      let cur_arg_list = [];
      let has_toc = false;
      for (const child_ast of content) {
        const macro_name = child_ast.macro_name;
        if (macro_name === Macro.TOC_MACRO_NAME) {
          has_toc = true;
        }
        if (
          macro_name === Macro.HEADER_MACRO_NAME &&
          // Just ignore extra added include headers, these
          // were overwritting index-split.html output.
          !child_ast.from_include &&
          !child_ast.validation_output.synonym.boolean
        ) {
          convert_header(cur_arg_list, context, has_toc);
          cur_arg_list = [];
          has_toc = false;
        }
        cur_arg_list.push(child_ast);
      }
      convert_header(cur_arg_list, context, has_toc);
      // Because the following conversion would redefine them.
    }
    perf_print(context, 'render_post')
    extra_returns.errors.push(...context.errors);
  }
  // Sort errors that might have been produced on different conversion
  // stages by line.
  extra_returns.errors = extra_returns.errors.sort((a, b)=>{
    if (a.source_location.path !== undefined && b.source_location.path !== undefined) {
      let ret = a.source_location.path.localeCompare(b.source_location.path);
      if (ret !== 0)
        return ret;
    }
    if (a.severity < b.severity)
      return -1;
    if (a.severity > b.severity)
      return 1;
    if (a.source_location.line < b.source_location.line)
      return -1;
    if (a.source_location.line > b.source_location.line)
      return 1;
    if (a.source_location.column < b.source_location.column)
      return -1;
    if (a.source_location.column > b.source_location.column)
      return 1;
    return 0;
  });
  if (output !== undefined) {
    if (output[output.length - 1] !== '\n') {
      output += '\n';
    }
  }
  perf_print(context, 'end_convert')
  return output;
}
exports.convert = convert;

/** Convert an argument to an XSS-safe output string.
 *
 * An argument contains a list of nodes, loop over that list of nodes,
 * converting them to strings and concatenate all strings.
 *
 * @param {AstArgument} arg
 * @return {String} empty string if arg is undefined
 */
function render_arg(arg, context) {
  let converted_arg = '';
  if (arg !== undefined) {
    for (const ast of arg) {
      converted_arg += ast.render(context);
    }
  }
  return converted_arg;
}

/* Similar to render_arg, but used for IDs.
 *
 * Because IDs are used programmatically in cirodown, we don't escape
 * HTML characters at this point.
 *
 * @param {AstArgument} arg
 * @return {String}
 */
function render_arg_noescape(arg, context={}) {
  return render_arg(arg, clone_and_set(context, 'html_escape', false));
}

/* Convert one header (or content before the first header)
 * in --split-headers mode. */
function convert_header(cur_arg_list, context, has_toc) {
  if (
    // Can fail if:
    // * the first thing in the document is a header
    // * the document has no headers
    cur_arg_list.length > 0
  ) {
    context = Object.assign({}, context);
    const options = Object.assign({}, context.options);
    context.options = options;
    const first_ast = clone_object(cur_arg_list[0]);
    if (!has_toc && first_ast.macro_name === Macro.HEADER_MACRO_NAME) {
      cur_arg_list.push(new AstNode(
        AstType.MACRO,
        Macro.TOC_MACRO_NAME,
        {},
        first_ast.source_location,
        {id: Macro.TOC_ID}
      ));
    }
    const ast_toplevel = new AstNode(
      AstType.MACRO,
      Macro.TOPLEVEL_MACRO_NAME,
      {
        'content': new AstArgument(cur_arg_list, first_ast.source_location)
      },
      first_ast.source_location,
    );
    context.toplevel_id = first_ast.id;
    context.in_split_headers = true;
    // When not in simple header mode, we always have a value-less node, with
    // children with values. Now things are a bit more complicated, because we
    // want to keep the header tree intact, but at the same time also uniquely point
    // to one of the headers. So let's fake a tree node that has only one child we care
    // about. And the child does not have this fake parent to be able to see actual parents.
    context.header_tree = new HeaderTreeNode();
    // Clone this, because we are going to modify it, and it would affect
    // non-split headers and final outputs afterwards.
    first_ast.header_tree_node = clone_object(first_ast.header_tree_node);
    context.header_tree.add_child(first_ast.header_tree_node);
    context.header_tree_top_level = first_ast.header_tree_node.get_level();
    const output_path = (output_path_from_ast(
      first_ast,
      clone_and_set(context, 'to_split_headers', true)
    ))[0];
    if (options.log['split-headers']) {
      console.error('split-headers: ' + output_path);
    }
    context.toplevel_output_path = output_path;
    context.toplevel_ast = first_ast;

    // root_relpath
    options.template_vars = Object.assign({}, options.template_vars);
    const new_root_relpath = get_root_relpath(output_path, context)
    context.root_relpath_shift = path.relative(options.template_vars.root_relpath,
      path.join(context.root_relpath_shift, new_root_relpath))
    options.template_vars.root_relpath = new_root_relpath

    // Do the conversion.
    context.extra_returns.rendered_outputs[output_path] =
      ast_toplevel.render(context);
  }
}

/**
 * @param {Object} options:
 *        - {Number} start_line
 *        - {Array} errors
 * @return {AstArgument}*/
async function parse_include(
  input_string,
  convert_options,
  cur_header_level,
  input_path,
  href,
  options={}
) {
  convert_options = Object.assign({}, convert_options);
  convert_options.from_include = true;
  convert_options.h_parse_level_offset = cur_header_level;
  convert_options.input_path = input_path;
  convert_options.render = false;
  convert_options.toplevel_id = href;
  convert_options.header_tree_stack = new Map(convert_options.header_tree_stack);
  convert_options.header_tree_id_stack = new Map(convert_options.header_tree_id_stack);
  if (options.start_line !== undefined) {
    convert_options.start_line = options.start_line;
  }
  const convert_extra_returns = {};
  await convert(
    input_string,
    convert_options,
    convert_extra_returns,
  );
  if (options.errors !== undefined) {
    options.errors.push(...convert_extra_returns.errors);
  }
  return convert_extra_returns.ast.args.content;
}

function convert_init_context(options={}, extra_returns={}) {
  options = Object.assign({}, options);
  if (!('add_test_instrumentation' in options)) { options.add_test_instrumentation = false; }
  if (!('body_only' in options)) { options.body_only = false; }
  if (!('cirodown_json' in options)) { options.cirodown_json = {}; }
    const cirodown_json = options.cirodown_json;
    {
      if (!('media-providers' in cirodown_json)) { cirodown_json['media-providers'] = {}; }
      {
        const media_providers = cirodown_json['media-providers'];
        for (const media_provider_type of MEDIA_PROVIDER_TYPES) {
          if (!(media_provider_type in media_providers)) {
            media_providers[media_provider_type] = {};
          }
          const media_provider = media_providers[media_provider_type];
          if (!('title-from-src' in media_provider)) {
            media_provider['title-from-src'] = false;
          }
        }
        if (media_providers.local && !('path' in media_providers.local)) {
          media_providers.local.path = '';
        }
        if (media_providers.github && !('remote' in media_providers.github)) {
          media_providers.github.remote = 'TODO determine from git remote origin if any';
        }
        for (const media_provider_name in media_providers) {
          const media_provider = media_providers[media_provider_name];
          if (!('title-from-src' in media_provider)) {
            media_provider['title-from-src'] = false;
          }
        }
      }
      if (!('h' in cirodown_json)) { cirodown_json.h = {}; }
      if (!('numbered' in cirodown_json.h)) { cirodown_json.h.numbered = true; }
      if (!('splitDefault' in cirodown_json.h)) { cirodown_json.h.splitDefault = false; }
      {
        if (!('lint' in cirodown_json)) { cirodown_json.lint = {}; }
        const lint = cirodown_json.lint
        if (!('h-tag' in lint)) { lint['h-tag'] = undefined; }
        if (!('h-parent' in lint)) { lint['h-parent'] = undefined; }
      }
      {
        if (!('id' in cirodown_json)) { cirodown_json.id = {}; }
        const id = cirodown_json.id
        if (!('normalize' in id)) { id.normalize = {}; }
        const normalize = id.normalize
      }
    }
  if (!('embed_includes' in options)) { options.embed_includes = false; }
  if (!('file_provider' in options)) { options.file_provider = undefined; }
  // Check if file exists.
  if (!('fs_exists_sync' in options)) { options.fs_exists_sync }
  if (!('from_include' in options)) { options.from_include = false; }
  if (!('from_cirodown_example' in options)) { options.from_cirodown_example = false; }
  if (!('html_embed' in options)) { options.html_embed = false; }
  if (!('html_x_extension' in options)) {
    // Add HTML extension to x links. And therefore also:
    // * output files with the `.html` extension
    // * output `/index.html` vs just `/`
    options.html_x_extension = true;
  }
  if (!('h_parse_level_offset' in options)) {
    // When parsing, start the first header at this offset instead of h1.
    // This is used when doing includes, since the included header is at.
    // an offset relative to where it is included from.
    options.h_parse_level_offset = 0;
  }
  if (!('magic_leading_at' in options)) { options.magic_leading_at = true; }
  if (!('id_provider' in options)) { options.id_provider = undefined; }
  if (!('input_path' in options)) { options.input_path = undefined; }
  if (!('katex_macros' in options)) { options.katex_macros = {}; }
  if (!('log' in options)) { options.log = {}; }
  if (!('outfile' in options)) {
    // Override the default calculated output file for the main input.
    options.outfile = undefined;
  }
  if (!('output_format' in options)) { options.output_format = OUTPUT_FORMAT_HTML; }
  if (!('path_sep' in options)) { options.path_sep = undefined; }
  if (!('read_include' in options)) { options.read_include = () => undefined; }
  if (!('read_file' in options)) { options.read_file = () => undefined; }
  if (!('remove_leading_at' in options)) { options.remove_leading_at = false; }
  if (!('render' in options)) { options.render = true; }
  if (!('start_line' in options)) { options.start_line = 1; }
  if (!('split_headers' in options)) {
    options.split_headers = false;
  }
  if (!('template' in options)) { options.template = undefined; }
  if (!('template_scripts_relative' in options)) {
    // Like template_styles_relative but for sripts.
    options.template_scripts_relative = [];
  }
  if (!('template_styles_relative' in options)) {
    // CSS styles relative to cirodown.json. Must be resolved by cirodown.convert.
    // because of split headers. The relative path expanded result gets prepended
    // to `options.template_vars.style`.
    options.template_styles_relative = [];
  }
  if ('template_vars' in options) {
    options.template_vars = Object.assign({}, options.template_vars);
  } else {
    options.template_vars = {};
  }
    if (!('head' in options.template_vars)) { options.template_vars.head = ''; }
    if (!('root_relpath' in options.template_vars)) { options.template_vars.root_relpath = ''; }
    if (!('post_body' in options.template_vars)) { options.template_vars.post_body = ''; }
    if (!('style' in options.template_vars)) { options.template_vars.style = ''; }

  // Internalish options that may get modified by sub-includes/CirodownExample in order
  // to forward state back up. Maybe we should move them to a subdict to make this clearer
  // (moving to extra_returns feels bad because they are also input), but lazy.
  //
  // Non-indexed-ids: auto-generated numeric ID's like p-1, p-2, etc.
  // It is not possible to link to them from inside the document, since links
  // break across versions.
  if (!('include_path_set' in options)) { options.include_path_set = new Set(); }
  if (options.non_indexed_ids === undefined) {
    options.non_indexed_ids = {};
  }
  if (options.indexed_ids === undefined) {
    options.indexed_ids = {};
  }
  if (options.header_tree_stack === undefined) {
    options.header_tree_stack = new Map();
  }
  if (options.header_tree_id_stack === undefined) {
    options.header_tree_id_stack = new Map();
  }
  if (options.is_first_global_header === undefined) {
    options.is_first_global_header = true;
  }
  if (options.add_refs_to_h === undefined) {
    options.add_refs_to_h = [];
  }
  if (options.add_refs_to_x === undefined) {
    options.add_refs_to_x = [];
  }
  if (options.include_hrefs === undefined) {
    options.include_hrefs = {};
  }

  // Handle scope and IDs that are based on the input path:
  //
  // - toplevel_has_scope
  //
  //   Set for index files in subdirectories. Is equivalent to
  //   adding a {scope} to the toplevel header.
  //
  // - toplevel_parent_scope
  //
  //   Set for files in subdirectories. Means that the (faked non existent)
  //   parent toplevel header has {scope} set.
  //
  // - toplevel_id
  //
  //   If true, force the toplevel header to have this ID.
  //   Otherwise, derive the ID from the title.
  //   https://cirosantilli.com/cirodown#the-id-of-the-first-header-is-derived-from-the-filename
  options.toplevel_id = undefined;
  let root_relpath_shift
  if (options.input_path !== undefined) {
    const [input_dir, basename] = path_split(options.input_path, options.path_sep)
    const [basename_noext, ext] = path_splitext(basename)
    if (INDEX_FILE_BASENAMES_NOEXT.has(basename_noext)) {
      if (input_dir === '') {
        // https://cirosantilli.com/cirodown#the-toplevel-index-file
        options.toplevel_id = undefined;
      } else {
        // https://cirosantilli.com/cirodown#the-id-of-the-first-header-is-derived-from-the-filename
        options.toplevel_id = input_dir;
        options.toplevel_has_scope = true
        root_relpath_shift = input_dir
      }
      options.isindex = true
    } else {
      options.toplevel_id = basename_noext;
      options.isindex = false
    }
    if (input_dir === '') {
      options.toplevel_parent_scope = undefined;
    } else {
      options.toplevel_parent_scope = input_dir
    }
  }
  if (root_relpath_shift === undefined) {
    root_relpath_shift = ''
  }

  if (options.unsafe_xss === undefined) {
    const unsafe_xss = cirodown_json['unsafe-xss'];
    if (unsafe_xss !== undefined) {
      options.unsafe_xss = unsafe_xss;
    } else {
      options.unsafe_xss = false;
    }
  }
  extra_returns.errors = [];
  extra_returns.debug_perf = {};
  const context = {
    perf_prev: 0,
    katex_macros: Object.assign({}, options.katex_macros),
    in_split_headers: false,
    in_parse: false,
    errors: [],
    extra_returns,
    include_path_set: new Set(options.include_path_set),
    in_header: false,
    macros: macro_list_to_macros(),
    options,
    // Shifts in local \a links due to either:
    // - scope + split headers e.g. scope/notindex.html
    // - subdirectories
    root_relpath_shift,
    synonym_headers: new Set(),
    toplevel_id: options.toplevel_id,
    // Output path for the current rendering.
    // Gets modified by split headers to point to the split header output path of each split header.
    toplevel_output_path: options.outfile,
  };
  perf_print(context, 'start_convert')
  return context;
}

/* Like x_href, but called with options as convert,
 * so that we don't have to fake a complex context. */
function convert_x_href(target_id, options) {
  const context = convert_init_context(options);
  context.id_provider = options.id_provider;
  const target_id_ast = context.id_provider.get(target_id, context);
  if (target_id_ast === undefined) {
    return undefined
  } else {
    return x_href(target_id_ast, context);
  }
}
exports.convert_x_href = convert_x_href;

function dirname(str, sep) {
  return path_split(str, sep)[0];
}

function duplicate_id_error_message(id, path, line, column) {
  let message = `duplicate ID "${id}", previous one defined at `;
  if (path !== undefined) {
    message += `file "${path}" `;
  }
  message += `line ${line} column ${column}`;
  return message
}
exports.duplicate_id_error_message = duplicate_id_error_message

/** Error message to be rendered inside the generated output itself.
 *
 * If context is given, escape the message correctly for this context.
 *
 * @return {String}
 */
function error_message_in_output(msg, context) {
  let escaped_msg;
  if (context === undefined) {
    escaped_msg = msg;
  } else {
    escaped_msg = html_escape_context(context, msg);
  }
  return `[CIRODOWN_ERROR: ${escaped_msg}]`
}

// https://stackoverflow.com/questions/9461621/format-a-number-as-2-5k-if-a-thousand-or-more-otherwise-900
const FORMAT_NUMBER_APPROX_MAP = [
  { value: 1, symbol: "" },
  { value: 1E3, symbol: "k" },
  { value: 1E6, symbol: "M" },
  { value: 1E9, symbol: "G" },
  { value: 1E12, symbol: "T" },
  { value: 1E15, symbol: "P" },
  { value: 1E18, symbol: "E" }
];
function format_number_approx(num, digits) {
  if (digits === undefined) {
    digits = 0;
  }
  const rx = /\.0+$|(\.[0-9]*[1-9])0+$/;
  let i;
  for (i = FORMAT_NUMBER_APPROX_MAP.length - 1; i > 0; i--) {
    if (num >= FORMAT_NUMBER_APPROX_MAP[i].value) {
      break;
    }
  }
  return (num / FORMAT_NUMBER_APPROX_MAP[i].value).toFixed(digits).replace(rx, "$1") + FORMAT_NUMBER_APPROX_MAP[i].symbol;
}

function get_descendant_count(tree_node) {
  return [
    tree_node.descendant_count,
    tree_node.word_count,
    tree_node.word_count + tree_node.descendant_word_count
  ];
}

function get_descendant_count_html(context, tree_node, options) {
  if (options.long_style === undefined) {
    options.long_style = false;
  }
  if (options.show_descendant_count === undefined) {
    options.show_descendant_count = true;
  }
  const [descendant_count, word_count, descendant_word_count] = get_descendant_count(tree_node);
  let ret;
  let word_count_html = ''
  const icon = '<span class="fa-regular-400">\u{f075}</span>'
  const words_str = options.long_style ? `words: ` : ''
  if (descendant_word_count > 0 && (context.options.add_test_instrumentation || options.show_descendant_count)) {
    word_count_html += `<span title="word count for this node + all descendants">${icon} ${words_str}<span class="word-count-descendant">${format_number_approx(descendant_word_count)}</span></span>`
  }
  if (tree_node.word_count > 0 && context.options.add_test_instrumentation || !options.show_descendant_count) {
    word_count_html += `<span title="word count for this node">${icon} ${words_str}<span class="word-count">${format_number_approx(word_count)}</span></span>`;
  }
  if (descendant_count > 0 && context.options.add_test_instrumentation) {
    word_count_html += `<span class="descendant-count">${format_number_approx(descendant_count)}</span>`
  }
  if (word_count_html !== '') {
    ret = `<span class="metrics">${word_count_html}</span>`;
  }
  return ret;
}

function get_descendant_count_html_sep(context, tree_node, options) {
  let ret = get_descendant_count_html(context, tree_node, options);
  if (ret !== undefined) {
    ret = `<span class="metrics-sep">${HEADER_MENU_ITEM_SEP + ret}</span>`;
  }
  return ret;
}

function check_and_update_local_link({
  check,
  context,
  href,
  relative,
  source_location,
}) {
  const was_protocol_given = protocol_is_given(href)

  let input_path_directory
  if (context.options.input_path !== undefined) {
    input_path_directory = dirname(
      context.options.input_path,
      context.options.path_sep
    )
  } else {
    input_path_directory = '.'
  }

  // Check existence.
  let error = ''
  if (
    (check !== undefined && check) ||
    (check === undefined && !was_protocol_given)
  ) {
    if (href.length !== 0) {
      let check_path;
      if (href[0] === URL_SEP) {
        check_path = href.slice(1)
      } else {
        check_path = path.join(input_path_directory, href)
      }
      if (!context.options.fs_exists_sync(check_path)) {
        error = `link to inexistent local file: ${href}`;
        render_error(context, error, source_location);
      }
    }
  }
  if (error) {
    error = error_message_in_output(error, context)
  }

  // Modify relative paths to account for scope + --split-headers
  if (
    (relative !== undefined && relative) ||
    (
      relative === undefined
      && (
        !was_protocol_given &&
        (
          href.length === 0 ||
          href[0] !== URL_SEP
        )
      )
    )
  ) {
    href = path.join(context.root_relpath_shift, href);
  }
  return { href, error }
}

// Get description and other closely related attributes.
function get_description(description_arg, context) {
  let description = render_arg(description_arg, context);
  let force_separator
  if (description === '') {
    force_separator = false
  } else {
    description = ' ' + description;
    force_separator = true;
  }

  let multiline_caption
  if (description_arg) {
    for (const ast of description_arg) {
      if (!(
        context.macros[ast.macro_name].options.phrasing ||
        ast.node_type === AstType.PLAINTEXT
      )) {
        multiline_caption = ' multiline-caption'
        break
      }
    }
  }
  if (multiline_caption === undefined) {
    multiline_caption = ''
  }

  return { description, force_separator, multiline_caption }
}

function get_link_html({
  attrs,
  check,
  content,
  context,
  href,
  relative,
  source_location,
}) {
  if (context.x_parents.size === 0) {
    if (attrs === undefined) {
      attrs = ''
    }
    let error
    ;({ href, error } = check_and_update_local_link({
      check,
      context,
      href,
      relative,
      source_location,
    }))
    return `<a${html_attr('href', href)}${attrs}>${content}${error}</a>`;
  } else {
    // Don't create a link if we are a child of another link, as that is invalid HTML.
    return content;
  }
}

/** Get the AST from the parent argument of headers or includes. */
function get_parent_argument_ast(ast, context, prev_header, include_options) {
  let parent_id;
  let parent_ast;
  parent_id = render_arg_noescape(ast.args.parent, context);
  if (
    // Happens for the first header
    prev_header !== undefined
  ) {
    if (parent_id[0] === Macro.HEADER_SCOPE_SEPARATOR) {
      parent_ast = context.id_provider.get_noscope(parent_id.substr(1), context);
    } else {
      // We can't use context.id_provider.get here because we don't know who
      // the parent node is, because scope can affect that choice.
      // https://cirosantilli.com/cirodown#id-based-header-levels-and-scope-resolution
      let sorted_keys = [...include_options.header_tree_stack.keys()].sort((a, b) => a - b);
      let largest_level = sorted_keys[sorted_keys.length - 1];
      for (let level = largest_level; level > 0; level--) {
        let ast = include_options.header_tree_stack.get(level).ast;
        if (id_is_suffix(parent_id, ast.id)) {
          parent_ast = ast;
          break;
        }
      }
    }
  }
  return [parent_id, parent_ast];
}

function get_root_relpath(output_path, context) {
  // TODO htmlEmbed was split into embedIncludes and embedResources.
  // This was likely meant to be embedIncludes, but I don't have a filing test if this is commented out
  // so not sure.
  const [output_path_dir, output_path_basename] =
    path_split(output_path, context.options.path_sep);
  let root_relpath = path.relative(output_path_dir, '.')
  if (root_relpath !== '') {
    root_relpath += URL_SEP;
  }
  return root_relpath
}

// Ensure that all children and tag targets exist. This is for error checking only.
// https://cirosantilli.com/cirodown#h-child-argment
function header_check_child_tag_exists(ast, context, childrenOrTags, type) {
  let ret = ''
  for (let child of childrenOrTags) {
    const target_id = render_arg_noescape(child.args.content, context)
    const target_id_ast = context.id_provider.get(target_id, context, ast.header_tree_node)
    if (target_id_ast === undefined) {
      let message = `unknown ${type} id: "${target_id}"`
      render_error(context, message, child.source_location)
      ret += error_message_in_output(message, context)
    }
  }
  return ret
}

/** Convert a key value already fully HTML escaped strings
 * to an HTML attribute. The callers MUST escape any untrusted chars.
  e.g. with html_attr_value.
 *
 * @param {String} key
 * @param {AstArgument} arg
 * @return {String} - of form ' a="b"' (with a leading space)
 */
function html_attr(key, value) {
  return ` ${key}="${value}"`;
}

/** Convert an argument to an HTML attribute value.
 *
 * @param {AstArgument} arg
 * @param {Object} context
 * @return {String}
 */
function html_attr_value(arg, context) {
  return render_arg(arg, clone_and_set(context, 'html_is_attr', true));
}

function html_class_attr(classes) {
  return html_attr('class', classes.join(' '))
}

function html_code(content, attrs) {
  return html_elem('pre', html_elem('code', content), attrs);
}

/** Helper to convert multiple parameters directly to HTML attributes.
 *
 * The ID is automatically included.
 *
 * @param {AstNode} ast
 * @param {Object} options
 * @param {Array[String]} arg_names - which argument names should be added as properties.
 *         Only arguments that were given in the text input are used.
 * @param {Object[String, AstNode]} custom_args - attributes that were not just passed in
 *        directly from the input text, but may rather have been calculated from the node.
 */
function html_render_attrs(
  ast, context, arg_names=[], custom_args={}
) {
  // Determine the arguments.
  let args = [];
  for (const arg_name in custom_args) {
    args.push([arg_name, custom_args[arg_name]]);
  }
  for (const arg_name of arg_names) {
    if (arg_name in ast.args) {
      args.push([arg_name, ast.args[arg_name]]);
    }
  }

  // Build the output string.
  let ret = '';
  for (const name_arg_pair of args) {
    const [arg_name, arg] = name_arg_pair;
    ret += html_attr(arg_name, html_attr_value(arg, context));
  }
  return ret;
}

/**
 * Same interface as html_render_attrs, but automatically add the ID to the list
 * of arguments.
 */
function html_render_attrs_id(
  ast, context, arg_names=[], custom_args={}
) {
  let id = ast.id;
  if (id) {
    custom_args[Macro.ID_ARGUMENT_NAME] = [
      new PlaintextAstNode(
        remove_toplevel_scope(id, context.toplevel_ast, context),
        ast.source_location,
      ),
    ];
  }
  return html_render_attrs(ast, context, arg_names, custom_args);
}

/** Helper for the most common HTML function type that does "nothing magic":
 * only has "id" as a possible attribute, and uses ast.args.content as the
 * main element child.
 */
function html_render_simple_elem(elem_name, options={}) {
  if (!('attrs' in options)) {
    options.attrs = {};
  }
  if (!('newline_after_open' in options)) {
    options.newline_after_open = false;
  }
  if (!('newline_after_close' in options)) {
    options.newline_after_close = false;
  }
  if (!('wrap' in options)) {
    options.wrap = false;
  }
  let newline_after_open_str;
  if (options.newline_after_open) {
    newline_after_open_str = '\n';
  } else {
    newline_after_open_str = '';
  }
  let newline_after_close_str;
  if (options.newline_after_close) {
    newline_after_close_str = '\n';
  } else {
    newline_after_close_str = '';
  }
  return function(ast, context) {
    let attrs = html_render_attrs_id(ast, context);
    let extra_attrs_string = '';
    for (const key in options.attrs) {
      extra_attrs_string += html_attr(key, options.attrs[key]);
    }
    let content_ast = ast.args.content;
    let content = render_arg(content_ast, context);

    // testData
    let test_data_arg = ast.args[Macro.TEST_DATA_ARGUMENT_NAME]
    let test_data_attr;
    if (test_data_arg === undefined) {
      test_data_attr = ''
    } else {
      test_data_attr = html_attr(Macro.TEST_DATA_HTML_PROP, render_arg(test_data_arg, context))
    }

    let res = `<${elem_name}${extra_attrs_string}${attrs}${test_data_attr}>${newline_after_open_str}${content}</${elem_name}>${newline_after_close_str}`;
    if (options.wrap) {
      res = html_elem('div', res);
    }
    return res;
  };
}

function html_elem(tag, content, attrs) {
  let ret = '<' + tag;
  for (const attr_id in attrs) {
    ret += ' ' + attr_id + '="' + html_escape_attr(attrs[attr_id]) + '"'
  }
  return ret + '>' + content + '</' + tag + '>';
}

function html_escape_attr(str) {
  return html_escape_content(str)
    .replace(/"/g, '&quot;')
    .replace(/'/g, '&#039;')
  ;
}

function html_escape_content(str) {
  return str
    .replace(/&/g, '&amp;')
    .replace(/</g, '&lt;')
    .replace(/>/g, '&gt;')
  ;
}

/** Escape string depending on the current context. */
function html_escape_context(context, str) {
  if (context.html_escape) {
    if (context.html_is_attr) {
      return html_escape_attr(str);
    } else {
      return html_escape_content(str);
    }
  } else {
    return str;
  }
}

function html_hide_hover_link(id) {
  if (id === undefined) {
    return '';
  } else {
    let href = html_attr('href', html_escape_attr(id));
    return `<span class="hide-hover"><a${href}>${UNICODE_LINK}</a></span>`;
  }
}

function html_img({ alt, ast, context, rendered_attrs, src }) {
  let error
  ;({ href: src, error } = check_and_update_local_link({
    check: ast.validation_output.check.boolean,
    context,
    href: src,
    source_location: ast.args.src.source_location,
  }))
  return `<img${html_attr('src',
    html_escape_attr(src))
  }${html_attr('loading', 'lazy')}${rendered_attrs}${alt}>${error}\n`;
}

function html_is_whitespace_text_node(ast) {
  return ast.node_type === AstType.PLAINTEXT && html_is_whitespace(ast.text);
}

// https://stackoverflow.com/questions/2161337/can-we-use-any-other-tag-inside-ul-along-with-li/60885802#60885802
function html_is_whitespace(string) {
  for (const c of string) {
    if (!HTML_ASCII_WHITESPACE.has(c))
      return false;
  }
  return true;
}

function html_katex_convert(ast, context) {
  try {
    return katex.renderToString(
      render_arg(ast.args.content, clone_and_set(context, 'html_escape', false)),
      {
        globalGroup: true,
        macros: context.katex_macros,
        // The default is to also add MathML output for blind people.
        // However, it adds it with absolute positioning for some reason.
        // And as a result, if you add a math formula to the bottom of the editor,
        // it generates a toplevel scrollbar on Chromium 84 but not Firefox 79.
        output: 'html',
        strict: 'error',
        throwOnError: true,
      }
    );
  } catch(error) {
    // TODO remove the crap KaTeX adds to the end of the string.
    // It uses Unicode char hacks to add underlines... and there are two trailing
    // chars after the final newline, so the error message is taking up two lines
    let message = error.toString().replace(/\n\xcc\xb2$/, '');
    render_error(context, message, ast.args.content.source_location);
    return error_message_in_output(message, context);
  }
}

function html_self_link(ast, context) {
  return x_href_attr(
    ast,
    clone_and_set(context, 'to_split_headers', context.in_split_headers)
  );
}

/** https://stackoverflow.com/questions/14313183/javascript-regex-how-do-i-check-if-the-string-is-ascii-only/14313213#14313213 */
function is_ascii(str) {
  return /^[\x00-\x7F]*$/.test(str);
}

function id_convert_simple_elem(argname) {
  if (argname === undefined) {
    argname = 'content'
  }
  return function(ast, context) {
    let ret = render_arg(ast.args[argname], context);
    if (!context.macros[ast.macro_name].options.phrasing) {
      ret += '\n';
    }
    return ret;
  };
}

/** bb,      aaa/bbb -> false
 *  bbb,     aaa/bbb -> true
 *  aaa/bbb, aaa/bbb -> true
 */
function id_is_suffix(suffix, full) {
  let full_len = full.length;
  let suffix_len = suffix.length;
  return (
    full.endsWith(suffix) &&
    (
      full_len == suffix_len ||
      (
        full_len > suffix_len &&
        full[full_len - suffix_len - 1] === Macro.HEADER_SCOPE_SEPARATOR
      )
    )
  );
}

// @return [href: string, content: string ], both XSS safe.
function link_get_href_content(ast, context) {
  const href = render_arg(ast.args.href, clone_and_set(context, 'html_is_attr', true))
  let content = render_arg(ast.args.content, context);
  if (content === '') {
    content = render_arg(ast.args.href, context);
  }
  return [href, content];
}

// If in split header mode, link to the nosplit version.
// If in the nosplit mode, link to the split version.
function link_to_split_opposite(ast, context) {
  let content;
  let title;
  if (context.in_split_headers) {
    content = NOSPLIT_MARKER;
    title = 'view all headers in a single page';
  } else {
    content = SPLIT_MARKER;
    title = 'view one header per page';
  }
  let other_context = clone_and_set(context, 'to_split_headers', !context.in_split_headers);
  let other_href = x_href_attr(ast, other_context);
  return `<a${html_attr('title', title)}${other_href}>${content}</a>`;
}

/**
 * @return {Object} dict of macro name to macro
 */
function macro_list_to_macros() {
  const macros = {};
  for (const macro of macro_list()) {
    for (const format in MACRO_CONVERT_FUNCIONS) {
      macro.add_convert_function(format, MACRO_CONVERT_FUNCIONS[format][macro.name]);
    }
    macros[macro.name] = macro;
  }
  return macros;
}

/** At some point we will generalize this to on-the-fly macro definitions. */
function macro_list() {
  return DEFAULT_MACRO_LIST;
}
exports.macro_list = macro_list;

const CIRODOWN_EXT = '.ciro';
exports.CIRODOWN_EXT = CIRODOWN_EXT;
const MEDIA_PROVIDER_TYPES = new Set([
  'github',
  'local',
  'unknown',
  'wikimedia',
  'youtube',
]);
const media_provider_type_wikimedia_re = new RegExp('^https?://upload.wikimedia.org/wikipedia/commons/');
const media_provider_type_youtube_re = new RegExp('^https?://(www\.)?(youtube.com|youtu.be)/');
const macro_image_video_block_convert_function_wikimedia_source_url = 'https://commons.wikimedia.org/wiki/File:';
const macro_image_video_block_convert_function_wikimedia_source_image_re = new RegExp('^\\d+px-');
const macro_image_video_block_convert_function_wikimedia_source_video_re = new RegExp('^([^.]+\.[^.]+).*');

function macro_image_video_block_convert_function(ast, context) {
  let rendered_attrs = html_render_attrs(ast, context, ['height', 'width']);
  let figure_attrs = html_render_attrs_id(ast, context);
  let { description, force_separator, multiline_caption } = get_description(ast.args.description, context)
  let figure_class
  if (multiline_caption) {
    figure_class = html_attr('class', multiline_caption.slice(1))
  } else {
    figure_class = ''
  }
  let ret = `<figure${figure_attrs}${figure_class}>\n`
  let href_prefix;
  if (ast.id !== undefined) {
    href_prefix = html_self_link(ast, context);
  } else {
    href_prefix = undefined;
  }
  let {error_message, media_provider_type, source, src, is_url}
    = macro_image_video_resolve_params_with_source(ast, context);
  if (error_message !== undefined) {
    return error_message;
  }
  if (source !== '') {
    force_separator = true;
    source = ` <a${html_attr('href', source)}>Source</a>.`;
  }
  let alt_val;
  const has_caption = (ast.id !== undefined) && caption_number_visible(ast, context);
  if (ast.args.alt === undefined) {
    if (has_caption) {
      alt_val = undefined;
    } else {
      alt_val = src;
    }
  } else {
    alt_val = render_arg(ast.args.alt, context);
  }
  let alt;
  if (alt_val === undefined) {
    alt = '';
  } else {
    alt = html_attr('alt', html_escape_attr(alt_val));
  }
  ret += context.macros[ast.macro_name].options.image_video_content_func(
    ast, context, src, rendered_attrs, alt, media_provider_type, is_url);
  if (has_caption) {
    ret += `<figcaption>${x_text(ast, context, {href_prefix:
      href_prefix, force_separator})}${source}${description}</figcaption>\n`;
  }
  ret += '</figure>\n';
  return ret;
}

// https://stackoverflow.com/questions/44447847/enums-in-javascript-with-es6/49709701#49709701
function make_enum(arr) {
  let obj = {};
  for (let val of arr){
    obj[val] = Symbol(val);
  }
  return Object.freeze(obj);
}

function noext(str) {
  return str.substring(0, str.lastIndexOf('.'));
}

// https://cirosantilli.com/cirodown#ascii-normalization
function normalize_latin_character(c) {
  return lodash.deburr(c)
}

const NORMALIZE_PUNCTUATION_CHARACTER_MAP = {
  '+': 'plus',
  '&': 'and',
}
function normalize_punctuation_character(c) {
  if (c in NORMALIZE_PUNCTUATION_CHARACTER_MAP) {
    return ID_SEPARATOR + NORMALIZE_PUNCTUATION_CHARACTER_MAP[c] + ID_SEPARATOR
  } else {
    return c
  }
}

// https://stackoverflow.com/questions/17781472/how-to-get-a-subset-of-a-javascript-objects-properties/17781518#17781518
function object_subset(source_object, keys) {
  const new_object = {};
  keys.forEach((obj, key) => { new_object[key] = source_object[key]; });
  return new_object;
}

/** Calculate the output path of an AST node,
 * or of something faked to look like one.
 *
 * cirodown ->
 *    split_headers -> index.html
 *   !split_headers -> index.html
 * quick-start ->
 *    split_headers -> quick-start.html
 *   !split_headers -> index.html
 * not-readme -> not-readme.html
 *    split_headers -> not-readme.html
 *   !split_headers -> not-readme.html
 * h2 in not the README -> not-readme.html
 *    split_headers -> h2-in-not-the-readme.html
 *   !split_headers -> not-readme.html
 * not-readme-with-scope ->
 *    split_headers -> not-readme-with-scope.html
 *   !split_headers ->  not-readme-with-scope.html
 * not-readme-with-scope/h2 ->
 *    split_headers -> not-readme-with-scope/h2.html
 *   !split_headers ->  not-readme-with-scope.html
 * subdir ->
 *    split_headers -> subdir/index.html
 *   !html_x_extension -> subdir
 *   !split_headers -> subdir/index.html
 * subdir/subdir-h2
 */
function output_path(input_path, id, context={}, split_suffix=undefined) {
  const [dirname, basename] = output_path_parts(input_path, id, context, split_suffix);
  return [
    path_join(dirname, basename + '.' + HTML_EXT, context.options.path_sep),
    dirname,
    basename
  ];
}

/** Helper for when we have an actual AstNode. */
function output_path_from_ast(ast, context) {
  let split_suffix;
  if (ast.args.splitSuffix !== undefined) {
    split_suffix = render_arg(ast.args.splitSuffix, context);
  }
  return output_path(ast.source_location.path, ast.id, context, split_suffix);
}

/* This is the centerpiece of path calculation. It determines where a given header
 * will land considering:
 *
 * * README.ciro -> index.ciro renaming
 * * split header stuff
 *
 * Note that the links need to do additional processing to determine this.
 *
 * This function does not determine if something is in the current file
 * or not: it always assumes it is not.
 *
 * Return an array [dirname, basename without extension] of the
 * path to where an AST gets rendered to. */
function output_path_parts(input_path, id, context, split_suffix=undefined) {
  let ret = '';
  let custom_split_suffix;
  const [dirname, basename] = path_split(input_path, context.options.path_sep);
  const renamed_basename_noext = rename_basename(noext(basename));
  const ast = context.id_provider.get(id, context);

  // We are the first header, or something that comes before it.
  let first_header_or_before = false;
  if (ast === undefined) {
    return [dirname, renamed_basename_noext];
  } else {
    if (ast.macro_name === Macro.HEADER_MACRO_NAME) {
      id = ast.id;
    } else {
      id = ast.get_header_parent_ids(context).values().next().value;
    }
    if (ast.is_first_header_in_input_file) {
      first_header_or_before = true;
    }
  }

  // Calculate the base basename_ret and dirname_ret.
  let dirname_ret;
  let basename_ret;
  const [id_dirname, id_basename] = path_split(id, URL_SEP);
  const to_split_headers = is_to_split_headers(ast, context);
  if (
    first_header_or_before ||
    (
      context.to_split_headers !== undefined &&
      !context.to_split_headers
    )
  ) {
    // For toplevel elements in split header mode, we have
    // to take care of index and -split suffix.
    if (renamed_basename_noext === INDEX_BASENAME_NOEXT) {
      // basename_ret
      if (to_split_headers === ast.split_default) {
        // The name is just index.html.
        basename_ret = renamed_basename_noext;
      } else {
        // The name is split.html or nosplit.html.
        basename_ret = '';
      }
    } else {
      basename_ret = renamed_basename_noext;
    }
    dirname_ret = dirname;
  } else {
    if (to_split_headers) {
      // Non-toplevel elements in split header mode are simple,
      // the ID just gives the output path directly.
      dirname_ret = id_dirname;
      basename_ret = id_basename;
    } else {
      dirname_ret = dirname;
      basename_ret = renamed_basename_noext;
    }
  }

  const [dir_dir, dir_base] = path_split(dirname_ret, context.options.path_sep);
  if (basename_ret === INDEX_BASENAME_NOEXT && dirname) {
    const [dir_dir, dir_base] = path_split(dirname, context.options.path_sep);
    dirname_ret = dir_dir
    basename_ret = dir_base
  }

  // Add -split, -nosplit or custom suffixes to basename_ret.
  let suffix_to_add;
  if (split_suffix === undefined || split_suffix === '') {
    suffix_to_add = to_split_headers ? 'split' : 'nosplit';
  } else {
    suffix_to_add = split_suffix;
  }
  if (
    (
      to_split_headers &&
      (
        // To split.html
        (
          first_header_or_before &&
          !ast.split_default
        ) ||

        // User explcitly gave {splitSuffix}
        split_suffix !== undefined
      )
    ) ||
    // User gave {splitDefault}, so we link to nosplit.
    !to_split_headers && ast.split_default
  ) {
    if (basename_ret !== '') {
      basename_ret += '-';
    }
    basename_ret += suffix_to_add;
  }

  return [dirname_ret, basename_ret];
}
exports.output_path_parts = output_path_parts;

/** Parse tokens into the AST tree.
 *
 * @param {Array[Token]} tokens
 * @return {Object} extra_returns
 *         - {Array[ErrorMessage]} errors
 *         - {Object} ids
 * @return {AstNode}
 */
async function parse(tokens, options, context, extra_returns={}) {
  perf_print(context, 'parse_start')
  context.in_parse = true
  extra_returns.errors = [];
  let state = {
    extra_returns,
    i: 0,
    macros: context.macros,
    options: options,
    token: tokens[0],
    tokens: tokens,
  };
  // Get toplevel arguments such as {title=}, see https://cirosantilli.com/cirodown#toplevel
  const ast_toplevel_args = parse_argument_list(
    state, Macro.TOPLEVEL_MACRO_NAME, AstType.MACRO);
  if ('content' in ast_toplevel_args) {
    parse_error(state, `the toplevel arguments cannot contain an explicit content argument`, new SourceLocation(1, 1));
  }

  // Inject a maybe paragraph token after those arguments.
  const paragraph_token = new Token(TokenType.PARAGRAPH, state.token.source_location);
  tokens.splice(state.i, 0, paragraph_token);
  state.token = paragraph_token;

  // Parse the main part of the document as the content
  // argument toplevel argument.
  const ast_toplevel_content_arg = parse_argument(
    state, state.token.source_location);

  // Create the toplevel macro itself.
  const ast_toplevel = new AstNode(
    AstType.MACRO,
    Macro.TOPLEVEL_MACRO_NAME,
    Object.assign(ast_toplevel_args, {'content': ast_toplevel_content_arg}),
    new SourceLocation(1, 1),
  );
  if (state.token.type !== TokenType.INPUT_END) {
    parse_error(state, `unexpected tokens at the end of input`);
  }
  if (context.options.log['ast-pp-simple']) {
    console.error('ast-pp-simple: after parse');
    console.error(ast_toplevel.toString());
    console.error();
  }

  // Ast post process pass 1
  //
  // Post process the AST depth first minimally to support includes.
  //
  // This could in theory be done in a single pass with the next one,
  // but that is much more hard to implement and maintain, because we
  // have to stich togetegher internal structures to maintain the header
  // tree across the includer and included documents.
  //
  // Another possibility would be to do it in the middle of the initial parse,
  // but let's not complicate that further either, shall we?
  context.headers_with_include = [];
  context.header_tree = new HeaderTreeNode();
  perf_print(context, 'post_process_1')
  let prev_header;
  let cur_header_level;
  let first_header_level;
  let first_header;
  let header_tree_last_level;
  let toplevel_parent_arg = new AstArgument([], new SourceLocation(1, 1));
  let todo_visit = [[toplevel_parent_arg, ast_toplevel]];
  // IDs that are indexed: you can link to those.
  const line_to_id_array = [];
  context.line_to_id = function(line) {
    let index = binary_search(line_to_id_array,
      [line, undefined], binary_search_line_to_id_array_fn);
    if (index < 0) {
      index = -(index + 1)
    }
    if (index == line_to_id_array.length) {
      if (line_to_id_array.length > 0) {
        index = line_to_id_array.length - 1;
      } else {
        return undefined;
      }
    }
    return line_to_id_array[index][1];
  };
  let macro_count_global = 0
  const macro_counts = {};
  const macro_counts_visible = {};
  const headers_from_include = {}
  let cur_header_tree_node;
  let is_first_header = true;
  extra_returns.ids = options.indexed_ids;

  // Format:
  // refs_to[false][to_id][type]{from_id: { defined_at: Set[defined_at], child_index: Number }
  // refs_to[to][from_id][type]{to_id: { defined_at: Set[defined_at], child_index: Number }
  context.refs_to = {
    false: {},
    true: {},
  };
  let local_id_provider = new DictIdProvider(
    options.indexed_ids,
    context.refs_to,
  );
  let id_provider;
  if (options.id_provider !== undefined) {
    id_provider = new ChainedIdProvider(
      local_id_provider,
      options.id_provider
    );
  } else {
    id_provider = local_id_provider;
  }
  context.id_provider = id_provider;
  options.include_path_set.add(options.input_path);
  let header_file_preview_ast, header_file_preview_ast_next
  const title_ast_ancestors = []
  const header_title_ast_ancestors = []
  const header_ids = []
  while (todo_visit.length > 0) {
    const pop = todo_visit.pop();
    if (pop === 'pop_title_ast_ancestors') {
      title_ast_ancestors.pop()
      continue
    }
    if (pop === 'pop_header_title_ast_ancestors') {
      header_title_ast_ancestors.pop()
      continue
    }
    const [parent_arg, ast] = pop
    if (parent_arg.argument_name === Macro.TITLE_ARGUMENT_NAME) {
      const parent_ast = parent_arg.parent_node
      title_ast_ancestors.push(parent_ast)
      todo_visit.push('pop_title_ast_ancestors')
      if (parent_ast.macro_name === Macro.HEADER_MACRO_NAME) {
        header_title_ast_ancestors.push(parent_ast)
        todo_visit.push('pop_header_title_ast_ancestors')
      }
    }
    let parent_arg_push_after = []
    let parent_arg_push_before = []
    const macro_name = ast.macro_name;
    ast.from_include = options.from_include;
    ast.from_cirodown_example = options.from_cirodown_example;
    ast.source_location.path = options.input_path;
    if (macro_name === Macro.INCLUDE_MACRO_NAME) {
      const peek_ast = todo_visit[todo_visit.length - 1][1];
      if (peek_ast.node_type === AstType.PLAINTEXT && peek_ast.text === '\n') {
        todo_visit.pop();
      }
      const href = render_arg_noescape(ast.args.href, context);

      // \Include parent argument handling.
      let parent_ast;
      let parent_id;
      validate_ast(ast, context);
      if (ast.validation_output.parent.given) {
        [parent_id, parent_ast] = get_parent_argument_ast(ast, context, prev_header, options)
        if (parent_ast === undefined) {
          const message = Macro.INCLUDE_MACRO_NAME + ' ' + HEADER_PARENT_ERROR_MESSAGE + parent_id;
          const error_ast = new PlaintextAstNode(' ' + error_message_in_output(message), ast.source_location);
          error_ast.parent_node = ast.parent_node;
          parent_arg.push(error_ast);
          parse_error(state, message, ast.args.parent.source_location);
        }
      }
      if (parent_ast === undefined) {
        parent_ast = options.cur_header;
      }
      let parent_ast_header_level
      let parent_ast_header_tree_node
      let include_id = href
      if (options.cur_header && options.cur_header.scope) {
        include_id = options.cur_header.scope + Macro.HEADER_SCOPE_SEPARATOR + include_id
      }
      if (parent_ast === undefined) {
        parent_ast_header_level = 0
        parent_ast_header_tree_node = context.header_tree
      } else {
        // Possible on include without a parent header.
        parent_ast_header_tree_node = parent_ast.header_tree_node;
        parent_ast_header_level = parent_ast_header_tree_node.get_level();

        add_to_refs_to(
          include_id,
          context,
          parent_ast.id,
          REFS_TABLE_PARENT,
          parent_ast_header_tree_node.children.length
        );
        parent_ast.includes.push(href);
      }
      let input_dir, input_basename
      if (options.input_path) {
        ;[input_dir, input_basename] = path_split(options.input_path, options.path_sep)
      } else {
        input_dir = '.'
      }
      // https://github.com/cirosantilli/cirodown/issues/215
      const read_include_ret = await (options.read_include(href, input_dir));
      if (read_include_ret === undefined) {
        let message = `could not find include: "${href}"`;
        parse_error(
          state,
          message,
          ast.source_location,
        );
        parent_arg.push(new PlaintextAstNode(message, ast.source_location));
      } else {
        const [include_path, include_content] = read_include_ret;
        if (options.include_path_set.has(include_path)) {
          let message = `circular include detected to: "${include_path}"`;
          parse_error(
            state,
            message,
            ast.source_location,
          );
          parent_arg.push(new PlaintextAstNode(message, ast.source_location));
        } else {
          let new_child_nodes;
          if (options.embed_includes) {
            new_child_nodes = await parse_include(
              include_content,
              options,
              parent_ast_header_level,
              include_path,
              href,
              {
                errors: extra_returns.errors,
              }
            );
            options.include_path_set.add(include_path);
          } else {
            const from_include = true
            // Don't merge into a single file, render as a dummy header and an xref link instead.
            const header_ast = new AstNode(
              AstType.MACRO,
              Macro.HEADER_MACRO_NAME,
              {
                'level': new AstArgument(
                  [
                    new PlaintextAstNode(
                      (parent_ast_header_level + 1).toString(),
                    )
                  ],
                ),
                [Macro.TITLE_ARGUMENT_NAME]: new AstArgument( [
                    // Will be patched in later in order to group all DB queries at the end of parse,
                    // as this requires getting an ID from DB.
                    new PlaintextAstNode('TODO patchme')
                  ],
                ),
              },
              undefined,
              {
                force_no_index: true,
                from_include,
                id: include_id,
                level: parent_ast_header_level + 1,
              },
            );
            options.include_hrefs[include_id] = header_ast
            headers_from_include[include_id] = header_ast
            if (options.cur_header !== undefined) {
              header_ast.scope = options.cur_header.scope
            }
            header_ast.header_tree_node = new HeaderTreeNode(header_ast, parent_ast_header_tree_node);
            parent_ast_header_tree_node.add_child(header_ast.header_tree_node);
            new_child_nodes = [
              header_ast,
              new AstNode(
                AstType.PARAGRAPH,
              ),
              new AstNode(
                AstType.MACRO,
                Macro.PARAGRAPH_MACRO_NAME,
                {
                  'content': new AstArgument(
                    [
                      new AstNode(
                        AstType.MACRO,
                        Macro.X_MACRO_NAME,
                        {
                          'href': new AstArgument(
                            [
                              new PlaintextAstNode(href)
                            ],
                          ),
                          'content': new AstArgument(
                            [
                              new PlaintextAstNode(
                                'This section is present in another page, follow this link to view it.',
                              )
                            ],
                          ),
                        },
                        undefined,
                      ),
                    ],
                  ),
                },
                undefined,
              ),
              new AstNode(AstType.PARAGRAPH),
            ];
            for (const child_node of new_child_nodes) {
              child_node.set_source_location(ast.source_location)
              child_node.set_recursively({
                count_words: false,
                from_include,
              })
            }
          }
          // Push all included nodes, but don't recurse because:
          // - all child includes will be resolved on the sub-render call
          // - the current header level must not move, so that consecutive \Include
          //   calls won't nest into one another
          for (const new_child_node of new_child_nodes) {
            new_child_node.parent_node = ast.parent_node;
          }
          parent_arg.push(...new_child_nodes);
        }
      }
    } else if (macro_name === Macro.CIRODOWN_EXAMPLE_MACRO_NAME) {
      parent_arg.push(...[
        new AstNode(
          AstType.MACRO,
          Macro.CODE_MACRO_NAME.toUpperCase(),
          {'content': ast.args.content},
          ast.source_location,
        ),
        new AstNode(
          AstType.MACRO,
          Macro.PARAGRAPH_MACRO_NAME,
          {
            'content': new AstArgument(
              [
                new PlaintextAstNode(
                  'which renders as:',
                  ast.source_location,
                )
              ],
              ast.source_location
            ),
          },
          ast.source_location,
        ),
        new AstNode(
          AstType.MACRO,
          'Q',
          {
            'content': await parse_include(
              render_arg_noescape(ast.args.content, context),
              clone_and_set(options, 'from_cirodown_example', true),
              0,
              options.input_path,
              undefined,
              {
                start_line: ast.source_location.line + 1,
                errors: extra_returns.errors,
              }
            )
          },
          ast.source_location,
        ),
      ]);
    } else {
      // Not CirodownExample.
      if (macro_name === Macro.HEADER_MACRO_NAME) {
        if (is_first_header) {
          ast.id = context.toplevel_id
          if (options.toplevel_has_scope) {
            // We also need to fake an argument here, because that will
            // get serialized to the database, which is needed for
            // toplevel scope removal from external links.
            const scope_arg = new AstArgument([], ast.source_location);
            ast.add_argument('scope', scope_arg);
          }
          if (options.toplevel_parent_scope !== undefined) {
            ast.scope = options.toplevel_parent_scope;
            if (!options.isindex) {
              // We skip calculation of scoped IDs, so gotta do it here.
              ast.id = ast.scope + Macro.HEADER_SCOPE_SEPARATOR + ast.id;
            }
          }
          ast.is_first_header_in_input_file = true;
        }

        // Required by calculate_id.
        validate_ast(ast, context);

        const is_synonym = ast.validation_output.synonym.boolean;
        const header_level = parseInt(
          render_arg_noescape(ast.args.level, context)
        )

        // splitDefault propagation to children.
        if (ast.validation_output.splitDefault.given) {
          ast.split_default = ast.validation_output.splitDefault.boolean;
        } else if (options.cur_header !== undefined) {
          ast.split_default = options.cur_header.split_default;
        } else {
          ast.split_default = options.cirodown_json.h.splitDefault;
        }

        if (is_synonym) {
          if (options.cur_header === undefined) {
            const message = `the first header of an input file cannot be a synonym`;
            parse_error(state, message, ast.args.level.source_location);
          }
          if (header_level !== 1) {
            const message = `synonym headers must be h1, got: ${header_level}`;
            parse_error(state, message, ast.args.level.source_location);
          }
          ast.synonym = options.cur_header.id;
          if (ast.args.title2 !== undefined) {
            if (ast.args.title2.length > 0) {
              const message = `the title2 of synonym headers must be empty`;
              parse_error(state, message, ast.args.level.source_location);
            }
            options.cur_header.title2s.push(ast);
          }
          context.synonym_headers.add(ast);
        } else {
          prev_header = options.cur_header;
          options.cur_header = ast;
          cur_header_level = header_level + options.h_parse_level_offset;
        }

        let parent_tree_node_error = false;
        let parent_id;
        if (ast.validation_output.parent.given) {
          let parent_ast;
          if (is_synonym) {
            const message = `synonym and parent are incompatible`;
            parse_error(state, message, ast.args.level.source_location);
          }
          if (header_level !== 1) {
            const message = `header with parent argument must have level equal 1`;
            ast.args[Macro.TITLE_ARGUMENT_NAME].push(
              new PlaintextAstNode(' ' + error_message_in_output(message), ast.source_location)
            );
            parse_error(state, message, ast.args.level.source_location);
          }
          [parent_id, parent_ast] = get_parent_argument_ast(ast, context, prev_header, options);
          let parent_tree_node;
          if (parent_ast !== undefined) {
            parent_tree_node = options.header_tree_id_stack.get(parent_ast.id);
          }
          if (parent_tree_node === undefined) {
            parent_tree_node_error = true;
          } else {
            cur_header_level = parent_tree_node.get_level() + 1;
          }
        }
        if ('level' in ast.args) {
          // Hack the level argument of the final AST to match for consistency.
          ast.args.level = new AstArgument([
            new PlaintextAstNode(cur_header_level.toString(), ast.args.level.source_location)],
            ast.args.level.source_location);
        }

        // lint['h-parent']
        if (
          context.options.cirodown_json.lint['h-parent'] &&
          !ast.validation_output.synonym.boolean
        ) {
          let message;
          if (
            context.options.cirodown_json.lint['h-parent'] === 'parent' &&
            !ast.validation_output.parent.given &&
            !is_first_header
          ) {
            message = `no parent given with lint['h-parent'] = "parent"`;
          } else if (
            context.options.cirodown_json.lint['h-parent'] === 'number' &&
            ast.validation_output.parent.given
          ) {
            message = `parent given with lint['h-parent'] = "number"`;
          }
          if (message) {
            parse_error(state, message, ast.source_location);
            parent_arg_push_after.push(new PlaintextAstNode(error_message_in_output(message), ast.source_location));
          }
        }

        // lint['h-tag']
        if (
          context.options.cirodown_json.lint['h-tag'] !== undefined
        ) {
          let message;
          let arg;
          if (
            context.options.cirodown_json.lint['h-tag'] === 'child' &&
            ast.validation_output[Macro.HEADER_TAG_ARGNAME].given
          ) {
            message = `tag given with lint['h-tag'] = "child"`;
            arg = ast.args[Macro.HEADER_TAG_ARGNAME]
          } else if (
            context.options.cirodown_json.lint['h-tag'] === 'tag' &&
            ast.validation_output[Macro.HEADER_CHILD_ARGNAME].given
          ) {
            message = `child given with lint['h-tag'] = "tag"`;
            arg = ast.args[Macro.HEADER_CHILD_ARGNAME]
          }
          if (message) {
            parse_error(state, message, arg.source_location);
            parent_arg_push_after.push(new PlaintextAstNode(error_message_in_output(message), arg.source_location));
          }
        }

        is_first_header = false;

        if (options.is_first_global_header) {
          first_header = ast;
          first_header_level = cur_header_level;
          header_tree_last_level = cur_header_level - 1;
          options.header_tree_stack.set(header_tree_last_level, context.header_tree);
          options.is_first_global_header = false;
        }
        let header_level_skip_error;
        if (is_synonym) {
          ast.scope = options.cur_header.scope;
        } else {
          cur_header_tree_node = new HeaderTreeNode(ast, options.header_tree_stack.get(cur_header_level - 1));
          if (cur_header_level - header_tree_last_level > 1) {
            header_level_skip_error = header_tree_last_level;
          }
          if (cur_header_level < first_header_level) {
            parse_error(
              state,
              `header level ${cur_header_level} is smaller than the level of the first header of the document ${first_header_level}`,
              ast.args.level.source_location,
            );
          }
          const parent_tree_node = options.header_tree_stack.get(cur_header_level - 1);
          if (parent_tree_node !== undefined) {
            parent_tree_node.add_child(cur_header_tree_node);
            const parent_ast = parent_tree_node.ast;
            if (parent_ast !== undefined) {
              let scope = calculate_scope(parent_ast);
              // The ast might already have a scope here through less common means such as
              // being in a subdirectory.
              if (ast.scope) {
                if (scope === undefined) {
                  scope = ''
                } else {
                  scope += Macro.HEADER_SCOPE_SEPARATOR
                }
                scope += ast.scope
              }
              ast.scope = scope;
            }
          }
          const old_tree_node = options.header_tree_stack.get(cur_header_level);
          options.header_tree_stack.set(cur_header_level, cur_header_tree_node);
          if (
            // Possible on the first insert of a level.
            old_tree_node !== undefined &&
            // Possible if the level is not an integer.
            old_tree_node.ast !== undefined
          ) {
            options.header_tree_id_stack.delete(old_tree_node.ast.id);
          }
          header_tree_last_level = cur_header_level;
        }
        ast.header_tree_node = cur_header_tree_node

        // Must come after the header tree step is mostly done, because scopes influence ID,
        // and they also depend on the parent node.
        ;({ macro_count_global } = calculate_id(ast, context, options.non_indexed_ids, options.indexed_ids,
          macro_counts, macro_count_global, macro_counts_visible, state, true, line_to_id_array));

        // Now stuff that must come after calculate_id.

        header_ids.push(ast.id)

        // https://github.com/cirosantilli/cirodown/issues/100
        if (parent_tree_node_error) {
          const message = HEADER_PARENT_ERROR_MESSAGE + parent_id;
          ast.args[Macro.TITLE_ARGUMENT_NAME].push(
            new PlaintextAstNode(' ' + error_message_in_output(message), ast.source_location));
          parse_error(state, message, ast.args.parent.source_location);
        }
        if (!is_synonym) {
          if (header_level_skip_error !== undefined) {
            const message = `skipped a header level from ${header_level_skip_error} to ${cur_header_level}`;
            ast.args[Macro.TITLE_ARGUMENT_NAME].push(
              new PlaintextAstNode(' ' + error_message_in_output(message), ast.source_location));
            parse_error(state, message, ast.args.level.source_location);
          }
          options.header_tree_id_stack.set(cur_header_tree_node.ast.id, cur_header_tree_node);
        }

        // Handle the H file argument previews.
        header_file_preview_ast = header_file_preview_ast_next
        header_file_preview_ast_next = undefined
        if (ast.file) {
          if (IMAGE_EXTENSIONS.has(path_splitext(ast.file)[1])) {
            header_file_preview_ast_next = new AstNode(
              AstType.MACRO,
              'Image',
              {
                'src': new AstArgument(
                  [
                    new PlaintextAstNode(ast.file)
                  ],
                ),
              },
            )
          } else if (
            VIDEO_EXTENSIONS.has(path_splitext(ast.file)[1]) ||
            ast.file.match(media_provider_type_youtube_re)
          ) {
            header_file_preview_ast_next = new AstNode(
              AstType.MACRO,
              'Video',
              {
                'src': new AstArgument(
                  [
                    new PlaintextAstNode(ast.file)
                  ],
                ),
              },
            )
          } else {
            // This works. We are not enabling it for now for the following reasons:
            // - what is a binary file or not?
            // - do we really want to duplicate huge files? How large is huge? iframe does not work well, e.g.
            //   files that would be downloaded like .yml are also downloaded from the iframe.
            //const protocol = protocol_get(ast.file)
            //if (protocol === null || protocol === 'file') {
            //  const content = context.options.read_file(ast.file, context)
            //  console.error({content});
            //  console.error(/[\x00-\x10]/.test(content));
            //  if (
            //    content !== undefined &&
            //    // https://stackoverflow.com/questions/1677644/detect-non-printable-characters-in-javascript
            //    !/[\x00]/.test(content)
            //  ) {
            //    header_file_preview_ast_next = new AstNode(
            //      AstType.MACRO,
            //      Macro.CODE_MACRO_NAME.toUpperCase(),
            //      {
            //        'content': new AstArgument(
            //          [
            //            new PlaintextAstNode(content)
            //          ],
            //        ),
            //      },
            //    )
            //  }
            //}
          }
          if (header_file_preview_ast_next) {
            header_file_preview_ast_next.set_source_location(ast.source_location)
          }
        }
        if (header_file_preview_ast) {
          parent_arg_push_before.push(header_file_preview_ast);
        }

        if (parent_arg_push_before.length) {
          parent_arg_push_before = parent_arg_push_before.concat([new AstNode(
            AstType.PARAGRAPH, undefined, undefined, ast.source_location)
          ])
        }

        // Push a paragraph separator after the header when adding nodes after.
        if (parent_arg_push_after.length) {
          parent_arg_push_after = [new AstNode(
            AstType.PARAGRAPH, undefined, undefined, ast.source_location
          )].concat(parent_arg_push_after)
        }

        // Add children/tags to the child database.
        // https://cirosantilli.com/cirodown#h-child-argment
        const children = ast.args[Macro.HEADER_CHILD_ARGNAME]
        if (children !== undefined) {
          for (let child of children) {
            options.add_refs_to_h.push({ ast, child: true, content: child.args.content, type: REFS_TABLE_X_CHILD })
          }
        }
        const tags = ast.args[Macro.HEADER_TAG_ARGNAME]
        if (tags !== undefined) {
          for (let tag of tags) {
            options.add_refs_to_h.push({ ast, child: false, content: tag.args.content, type: REFS_TABLE_X_CHILD })
          }
        }
      } else if (macro_name === Macro.X_MACRO_NAME) {
        if (header_title_ast_ancestors.length > 0 && ast.args.content === undefined) {
          const message = 'x without content inside title of a header: https://cirosantilli.com/cirodown#x-within-title-restrictions'
          ast.args.content = new AstArgument(
            [ new PlaintextAstNode(' ' + error_message_in_output(message), ast.source_location) ],
            ast.source_location
          );
          parse_error(state, message, ast.source_location);
        }
        options.add_refs_to_x.push({
          ast,
          title_ast_ancestors: Object.assign([], title_ast_ancestors),
          target_id: render_arg_noescape(ast.args.href, context),
        })
      }

      // Push this node into the parent argument list.
      // This allows us to skip nodes, or push multiple nodes if needed.
      parent_arg.push(...parent_arg_push_before);
      parent_arg.push(ast);
      parent_arg.push(...parent_arg_push_after);

      // Recurse.
      for (const arg_name in ast.args) {
        const arg = ast.args[arg_name];
        for (let i = arg.length() - 1; i >= 0; i--) {
          todo_visit.push([arg, arg.get(i)]);
        }
        // We make the new argument be empty so that children can
        // decide if they want to push themselves or not.
        arg.reset()
      }
    }
  }
  if (header_file_preview_ast_next) {
    ast_toplevel.args.content.push(header_file_preview_ast_next)
  }
  if (context.options.log['ast-pp-simple']) {
    console.error('ast-pp-simple: after pass 1');
    console.error(ast_toplevel.toString());
    console.error();
  }

  let fetch_header_tree_ids_rows
  let fetch_ancestors_rows
  const prefetch_file_ids = new Set()
  if (!options.from_include) {
    // Ast post process pass 2
    //
    // Post process the AST pre-order depth-first search after
    // inclusions are resolved to support things like:
    //
    // - the insane but necessary paragraphs double newline syntax
    // - automatic ul parent to li and table to tr
    // - remove whitespace only text children from ul
    // - extract all IDs into an ID index
    //
    // Normally only the toplevel includer will enter this code section.
    perf_print(context, 'post_process_2')
    // Calculate header_tree_top_level.
    //
    // - if aa header of this level is present in the document,
    //   there is only one of it. This implies for example that
    //   it does not get numerical prefixes like "1.2.3 My Header".
    //   when rendering, and it does not show up in the ToC.
    if (context.header_tree.children.length === 1) {
      context.header_tree_top_level = first_header_level;
      const toplevel_header_node = context.header_tree.children[0];
      const toplevel_header_ast = toplevel_header_node.ast;
      if (toplevel_header_node.children.length > 0) {
        toplevel_header_node.children[0].ast.toc_header = true;
      }
      context.toplevel_ast = toplevel_header_ast;
    } else {
      context.header_tree_top_level = first_header_level - 1;
      context.toplevel_ast = undefined;
      if (context.header_tree.children.length > 0) {
        context.header_tree.children[0].ast.toc_header = true;
      }
    }
    // Not modified by split headers.
    context.nosplit_toplevel_ast = context.toplevel_ast
    context.has_toc = false;
    let toplevel_parent_arg = new AstArgument([], new SourceLocation(1, 1));
    {
      const todo_visit = [[toplevel_parent_arg, ast_toplevel]];
      while (todo_visit.length > 0) {
        let [parent_arg, ast] = todo_visit.pop();
        const macro_name = ast.macro_name;
        const macro = context.macros[macro_name];

        if (macro_name === Macro.TOC_MACRO_NAME) {
          if (ast.from_include || context.has_toc) {
            // Skip.
            continue;
          }
          context.has_toc = true;
        } else if (
          macro_name === Macro.TOPLEVEL_MACRO_NAME &&
          ast.parent_node !== undefined
        ) {
          // Prevent this from happening. When this was committed originally,
          // it actually worked and output an `html` inside another `html`.
          // Maybe we could do something with iframe, but who cares about that?
          const message = `the "${Macro.TOPLEVEL_MACRO_NAME}" cannot be used explicitly`;
          ast = new PlaintextAstNode(error_message_in_output(message), ast.source_location);
          parse_error(state, message, ast.source_location);
        } else if (
          ast.macro_name === Macro.HEADER_MACRO_NAME
        ) {
          propagate_numbered(ast, context)
        }
        // Dump index of headers with includes.
        if (ast.includes.length > 0) {
          context.headers_with_include.push(ast);
        }
        if (
          // These had been validated earlier during header processing.
          macro_name === Macro.HEADER_MACRO_NAME
        ) {
          if (
            !context.has_toc &&
            ast.toc_header
          ) {
            parent_arg.push(new AstNode(
              AstType.MACRO,
              Macro.TOC_MACRO_NAME,
              {},
              new SourceLocation(undefined, undefined, context.options.input_path),
              {
                parent_node: ast.parent_node
              }
            ))
            context.has_toc = true;
          }
        } else {
          if (
            // This only happens in one known case: when you have include without embed,
            // where we do a context.id_provider.get, and we set the attributes to it
            // and the thing comes out of serialization validated.
            !ast.validated
          ) {
            validate_ast(ast, context);
          }
        }

        // Push this node into the parent argument list.
        // This allows us to skip nodes, or push multiple nodes if needed.
        parent_arg.push(ast);

        // Loop over the child arguments. We do this rather than recurse into them
        // to be able to easily remove or add nodes to the tree during this AST
        // post-processing.
        //
        // Here we do sibling-type transformations that need to loop over multiple
        // direct children in one go, such as:
        //
        // - auto add ul to li
        // - remove whitespace only text children from ul
        for (const arg_name in ast.args) {
          // The following passes consecutively update arg.
          let arg = ast.args[arg_name];
          let macro_arg = macro.name_to_arg[arg_name];

          // Handle elide_link_only.
          if (
            // Possible for error nodes.
            macro_arg !== undefined &&
            macro_arg.elide_link_only &&
            arg.length() === 1 &&
            arg.get(0).macro_name === Macro.LINK_MACRO_NAME
          ) {
            const href_arg = arg.get(0).args.href;
            href_arg.parent_node = ast;
            arg = href_arg;
          }

          // Child loop that adds table tr implicit parents to th and td.
          // This needs to be done on a separate pass before the tr implicit table adding.
          // It is however very similar to the other loop: the only difference is that we eat up
          // a trailing paragraph if followed by another.
          {
            const new_arg = new AstArgument([], arg.source_location);
            for (let i = 0; i < arg.length(); i++) {
              let child_node = arg.get(i);
              let new_child_nodes = [];
              let new_child_nodes_set = false;
              if (child_node.node_type === AstType.MACRO) {
                const child_macro_name = child_node.macro_name;
                if (
                  child_macro_name == Macro.TD_MACRO_NAME ||
                  child_macro_name == Macro.TH_MACRO_NAME
                ) {
                  const auto_parent_name = Macro.TR_MACRO_NAME;
                  const auto_parent_name_macro = state.macros[auto_parent_name];
                  if (
                    ast.macro_name !== auto_parent_name
                  ) {
                    const start_auto_child_node = child_node;
                    const new_arg_auto_parent = new AstArgument([], child_node.source_location);
                    while (i < arg.length()) {
                      const arg_i = arg.get(i);
                      if (arg_i.node_type === AstType.MACRO) {
                        if (
                          arg_i.macro_name == Macro.TD_MACRO_NAME ||
                          arg_i.macro_name == Macro.TH_MACRO_NAME
                        ) {
                          new_arg_auto_parent.push(arg_i);
                        } else {
                          break;
                        }
                      } else if (arg_i.node_type === AstType.PARAGRAPH) {
                        if (i + 1 < arg.length()) {
                          const arg_i_next_macro_name = arg.get(i + 1).macro_name;
                          if (
                            arg_i_next_macro_name == Macro.TD_MACRO_NAME ||
                            arg_i_next_macro_name == Macro.TH_MACRO_NAME
                          ) {
                            // Ignore this paragraph, it is actually only a separator between two \tr.
                            i++;
                          }
                        }
                        break;
                      } else if (
                        auto_parent_name_macro.name_to_arg['content'].remove_whitespace_children &&
                        html_is_whitespace_text_node(arg_i)
                      ) {
                        // Ignore the whitespace node.
                      } else {
                        break;
                      }
                      i++;
                    }
                    new_child_nodes_set = true;
                    new_child_nodes = new AstArgument([new AstNode(
                      AstType.MACRO,
                      auto_parent_name,
                      {
                        'content': new_arg_auto_parent,
                      },
                      start_auto_child_node.source_location,
                      {
                        parent_node: child_node.parent_node
                      }
                    )], child_node.source_location);
                    // Because the for loop will advance past it.
                    i--;
                  }
                }
              }
              if (!new_child_nodes_set) {
                new_child_nodes = [child_node];
              }
              new_arg.push(...new_child_nodes);
            }
            arg = new_arg;
          }

          // Child loop that adds ul and table implicit parents.
          {
            const new_arg = new AstArgument([], arg.source_location);
            for (let i = 0; i < arg.length(); i++) {
              let child_node = arg.get(i);
              let new_child_nodes = [];
              let new_child_nodes_set = false;
              if (
                (arg_name in macro.name_to_arg) &&
                macro.name_to_arg[arg_name].remove_whitespace_children &&
                html_is_whitespace_text_node(child_node)
              ) {
                new_child_nodes_set = true;
              } else if (child_node.node_type === AstType.MACRO) {
                let child_macro_name = child_node.macro_name;
                let child_macro = state.macros[child_macro_name];
                if (child_macro.auto_parent !== undefined) {
                  // Add ul and table implicit parents.
                  const auto_parent_name = child_macro.auto_parent;
                  const auto_parent_name_macro = state.macros[auto_parent_name];
                  if (
                    ast.macro_name !== auto_parent_name &&
                    !child_macro.auto_parent_skip.has(ast.macro_name)
                  ) {
                    const start_auto_child_node = child_node;
                    const new_arg_auto_parent = new AstArgument([], child_node.source_location);
                    while (i < arg.length()) {
                      const arg_i = arg.get(i);
                      if (arg_i.node_type === AstType.MACRO) {
                        if (state.macros[arg_i.macro_name].auto_parent === auto_parent_name) {
                          new_arg_auto_parent.push(arg_i);
                        } else {
                          break;
                        }
                      } else if (
                        auto_parent_name_macro.name_to_arg['content'].remove_whitespace_children &&
                        html_is_whitespace_text_node(arg_i)
                      ) {
                        // Ignore the whitespace node.
                      } else {
                        break;
                      }
                      i++;
                    }
                    new_child_nodes_set = true;
                    new_child_nodes = new AstArgument([new AstNode(
                      AstType.MACRO,
                      auto_parent_name,
                      {
                        'content': new_arg_auto_parent,
                      },
                      start_auto_child_node.source_location,
                      {
                        parent_node: child_node.parent_node
                      }
                    )], child_node.source_location);
                    // Because the for loop will advance past it.
                    i--;
                  }
                }
              }
              if (!new_child_nodes_set) {
                new_child_nodes = [child_node];
              }
              new_arg.push(...new_child_nodes);
            }
            arg = new_arg;
          }

          // Child loop that adds paragraphs.
          {
            let paragraph_indexes = [];
            for (let i = 0; i < arg.length(); i++) {
              const child_node = arg.get(i);
              if (child_node.node_type === AstType.PARAGRAPH) {
                paragraph_indexes.push(i);
              }
            }
            if (paragraph_indexes.length > 0) {
              const new_arg = new AstArgument([], arg.source_location);
              if (paragraph_indexes[0] > 0) {
                parse_add_paragraph(state, ast, new_arg, arg, 0, paragraph_indexes[0], options);
              }
              let paragraph_start = paragraph_indexes[0] + 1;
              for (let i = 1; i < paragraph_indexes.length; i++) {
                const paragraph_index = paragraph_indexes[i];
                parse_add_paragraph(state, ast, new_arg, arg, paragraph_start, paragraph_index, options);
                paragraph_start = paragraph_index + 1;
              }
              if (paragraph_start < arg.length()) {
                parse_add_paragraph(state, ast, new_arg, arg, paragraph_start, arg.length(), options);
              }
              arg = new_arg;
            }
          }

          // Push children to continue the search. We make the new argument be empty
          // so that children can decide if they want to push themselves or not.
          {
            const new_arg = new AstArgument([], arg.source_location);
            const macro_arg_count_words = macro_arg !== undefined && macro_arg.count_words
            for (let i = arg.length() - 1; i >= 0; i--) {
              const child_ast = arg.get(i)

              // Propagate count_words.
              if (!child_ast.count_words || !macro_arg_count_words) {
                child_ast.count_words = false
                child_ast.word_count = 0
              }

              todo_visit.push([new_arg, child_ast]);
            }
            // Update the argument.
            ast.args[arg_name] = new_arg;
          }
        }
      }
    }
    if (context.options.log['ast-pp-simple']) {
      console.error('ast-pp-simple: after pass 2');
      console.error(ast_toplevel.toString());
      console.error();
    }

    // Now do a pass that collects information that may be affected by
    // the tree modifications of the previous step, e.g. ID generation.
    perf_print(context, 'post_process_3')
    {
      const todo_visit = [ast_toplevel];
      while (todo_visit.length > 0) {
        const ast = todo_visit.pop();
        const macro_name = ast.macro_name;
        const macro = context.macros[macro_name];

        let children_in_header;
        if (macro_name === Macro.HEADER_MACRO_NAME) {
          // TODO start with the toplevel.
          cur_header_tree_node = ast.header_tree_node;
          children_in_header = true;
        } else {
          ast.header_tree_node = new HeaderTreeNode(ast, cur_header_tree_node);
          if (ast.in_header) {
            children_in_header = true;
          } else {
            if (cur_header_tree_node !== undefined) {
              cur_header_tree_node.word_count += ast.word_count;
            }
            children_in_header = false;
          }
          if (cur_header_tree_node !== undefined) {
            ast.scope = calculate_scope(cur_header_tree_node.ast);
          }

          // Header IDs already previously calculated for parent= so we don't redo it in that case.
          let ret = calculate_id(ast, context, options.non_indexed_ids, options.indexed_ids, macro_counts,
            macro_count_global, macro_counts_visible, state, false, line_to_id_array);
          macro_count_global = ret.macro_count_global

          // Propagate some header properties to non-header children.
          // This allows us to save some extra DB fetches, at the cost of making the JSON slightly larger.
          // and duplicating data a bit. But whaterver, simpler code with less JOINs.
          if (ast.header_tree_node.parent_node !== undefined && ast.header_tree_node.parent_node.ast !== undefined) {
            const header_ast = ast.header_tree_node.parent_node.ast
            ast.split_default = header_ast.split_default;
            ast.is_first_header_in_input_file = header_ast.is_first_header_in_input_file;
          }
        }

        // Push children to continue the search. We make the new argument be empty
        // so that children can decide if they want to push themselves or not.
        for (const arg_name in ast.args) {
          const arg = ast.args[arg_name];
          for (let i = arg.length() - 1; i >= 0; i--) {
            const ast = arg.get(i);
            ast.in_header = children_in_header;
            todo_visit.push(ast);
          }
        }
      }
    }
    if (context.options.log['ast-pp-simple']) {
      console.error('ast-pp-simple: after pass 3');
      console.error(ast_toplevel.toString());
      console.error();
    }

    perf_print(context, 'db_queries')

    let id_conflict_asts = []
    if (options.id_provider !== undefined) {
      const prefetch_ids = new Set()
      for (const ref of options.add_refs_to_h) {
        const id = render_arg_noescape(ref.content, context)
        prefetch_ids.add(id)
        ref.target_id = id
      }
      for (const ref of options.add_refs_to_x) {
        const ast = ref.ast
        const id = ref.target_id

        // We are going to walk up the scope tree and try to fetch
        // everything as we just can't know which level is the correct
        // level. Previously, we would do a query, go up, query, go up,
        // interactively until found, but that is not possible anymore that
        // we have grouped all queries at one point.
        let ids = []
        push_scope_resolution(ids, ast.scope, id)

        // We need the target IDs of any x to render it.
        for (const id of ids) {
          prefetch_ids.add(id)
          prefetch_file_ids.add(id)
        }
      }
      for (const id in options.include_hrefs) {
        // We need the target it to be able to render the dummy include title
        // with link to the real content.
        prefetch_ids.add(id)
      }

      // QUERRY EVERYTHING AT ONCE NOW!
      let get_noscopes_base_fetch, get_refs_to_fetch, get_refs_to_fetch_reverse
      ;[
        get_noscopes_base_fetch,
        get_refs_to_fetch,
        get_refs_to_fetch_reverse,
        fetch_header_tree_ids_rows,
        fetch_ancestors_rows,
      ] = await Promise.all([
        options.id_provider.get_noscopes_base_fetch(
          Array.from(prefetch_ids),
          new Set(),
          context,
        ),

        // TODO merge the following two refs fetch into one single DB query. Lazy now.
        // Started prototype at: https://github.com/cirosantilli/cirodown/tree/merge-ref-cache
        // The annoying part is deciding what needs to go in which direction of the cache.
        options.id_provider.get_refs_to_fetch(
          [
            // These are needed to render each header.
            // Shows on parents.
            REFS_TABLE_PARENT,
            // Shows on tags.
            REFS_TABLE_X_CHILD,

            // This is needed for the Incoming links at the bottom of each output file.
            REFS_TABLE_X,
          ],
          header_ids,
          {
            context,
            ignore_paths_set: context.options.include_path_set,
          },
        ),
        // This is needed to generate the "tagged" at the end of each output file.
        options.id_provider.get_refs_to_fetch(
          [
            REFS_TABLE_X_CHILD,
          ],
          header_ids,
          {
            context,
            reversed: true,
            ignore_paths_set: context.options.include_path_set,
          }
        ),

        options.id_provider.fetch_header_tree_ids(
          options.include_hrefs,
        ),
        options.id_provider.fetch_ancestors(context.toplevel_id, context),
      ])
    }

    // Reconcile the dummy include header with our actual knowledge from the DB, e.g.:
    // * patch the ID of the include headers.
    // Has to be deferred here after DB fetch obviously because we need he DB data.
    // This is hair pulling stuff. There has to be a better way...
    for (const href in options.include_hrefs) {
      const header_ast = options.include_hrefs[href]
      const target_id_ast = context.id_provider.get(href, context, header_ast.scope);
      if (target_id_ast === undefined) {
        let message = `ID in include not found on database: "${href}", ` +
          `needed to calculate the cross reference title. Did you forget to convert all files beforehand?`;
        header_ast.args[Macro.TITLE_ARGUMENT_NAME].get(0).text = error_message_in_output(message)
        if (options.render) {
          parse_error(state, message, header_ast.source_location);
        }
      } else {
        if (target_id_ast.is_first_header_in_input_file) {
          // We want the rendered placeholder to use its parent numbered, so as to follow the includer's numbering scheme,
          // but the descendants to follow what they would actually render in the output as so they will show correctly on ToC.
          header_ast.add_argument('numbered', new AstArgument(
            [
              new PlaintextAstNode(context.options.cirodown_json.h.numbered ? '1' : '0', header_ast.source_location),
            ],
            header_ast.source_location,
          ))
        }
        header_ast.splitDefault = target_id_ast.splitDefault
        propagate_numbered(header_ast, context)
        header_ast.set_source_location(target_id_ast.source_location)
        header_ast.header_tree_node.update_ancestor_counts(target_id_ast.header_tree_node_word_count)
        for (const argname in target_id_ast.args) {
          if (
            // We have to patch the level of the target ID (1) do our new dummy one in the current tree.
            argname !== 'level' &&
            argname !== 'wiki' &&
            target_id_ast.validation_output[argname].given
          ) {
            header_ast.args[argname] = target_id_ast.args[argname]
          }
        }
      }
      // This is a bit nasty and duplicates the header processing code,
      // but it is a bit hard to factor them out since this is a magic include header,
      // and all includes and headers must be parsed concurrently since includes get
      // injected under the last header.
      validate_ast(header_ast, context);

      if (target_id_ast !== undefined) {
        // We modify the cache here to ensure that the header ID has the full header_tree_node, which
        // then gets feched from \x{full} (notably ToC) in order to show the link number there.
        //
        // Yes, this erase IDs that come from other Includes, but we don´t have a use case for that
        // right now, e.g. the placholder include header does not show parents.
        target_id_ast.header_tree_node = header_ast.header_tree_node
        target_id_ast.header_parent_ids = []
      }
    }
    let build_header_tree_asts
    if (options.id_provider !== undefined) {
      build_header_tree_asts = context.options.id_provider.build_header_tree(
        fetch_header_tree_ids_rows, { context })
      context.options.id_provider.fetch_ancestors_build_tree(
        fetch_ancestors_rows, context)
    }

    if (context.options.file_provider !== undefined) {
      const prefetch_files = new Set()
      // TODO convert this to a join with the above queries. Lazy.
      for (const prefetch_file_id of prefetch_file_ids) {
        const prefetch_ast = context.id_provider.get_noscope(prefetch_file_id, context)
        if (
          // Possible in some error cases.
          prefetch_ast !== undefined
        ) {
          prefetch_files.add(prefetch_ast.source_location.path)
        }
      }
      if (options.id_provider !== undefined) {
        for (const ast of build_header_tree_asts) {
          prefetch_files.add(ast.source_location.path)
        }
      }
      if (prefetch_files.size) {
        await context.options.file_provider.get_path_entry_fetch(Array.from(prefetch_files), context)
      }
    }

    // Now for some final operations that don't go over the entire Ast Tree.
    perf_print(context, 'post_process_4')

    // Setup refs DB.
    for (const ref of options.add_refs_to_h) {
      const target_id_effective = x_child_db_effective_id(
        ref.target_id,
        context,
        ref.ast
      )
      if (ref.child) {
        add_to_refs_to(target_id_effective, context, ref.ast.id, ref.type);
      } else {
        add_to_refs_to(ref.ast.id, context, target_id_effective, ref.type);
      }
    }
    for (const ref of options.add_refs_to_x) {
      const ast = ref.ast
      const target_id_effective = x_child_db_effective_id(
        ref.target_id,
        context,
        ast
      )
      const parent_id = ast.get_local_header_parent_id();
      if (
        // Happens on some special elements e.g. the ToC.
        parent_id !== undefined
      ) {
        // TODO add test and enable this possible fix.
        if (
          // We don't want the "This section is present in another page" to count as a link.
          !ast.from_include
        ) {
          // Update xref database for incoming links.
          const from_ids = add_to_refs_to(target_id_effective, context, parent_id, REFS_TABLE_X);
        }

        // Update xref database for child/parent relationships.
        {
          let toid, fromid;
          if (ast.validation_output.child.boolean) {
            fromid = parent_id;
            toid = target_id_effective;
          } else if (ast.validation_output.parent.boolean) {
            toid = parent_id;
            fromid = target_id_effective;
          }
          if (toid !== undefined) {
            add_to_refs_to(toid, context, fromid, REFS_TABLE_X_CHILD);
          }
        }
      }
      for (const title_ast of ref.title_ast_ancestors) {
        add_to_refs_to(target_id_effective, context, title_ast.id, REFS_TABLE_X_TITLE_TITLE);
      }
    }

    const first_toplevel_child = ast_toplevel.args.content.get(0);
    if (first_toplevel_child !== undefined) {
      first_toplevel_child.first_toplevel_child = true;
    }

    for (const id in headers_from_include) {
      const ast = headers_from_include[id]
      // This to ensure that the ast we get from \x will have a consistent
      // numbering with the local parent.
      // This code will likely be removed if we do:
      // https://github.com/cirosantilli/cirodown/issues/188
      const cached_ast = context.id_provider.get(ast.id, context)
      if (
        // Possible in error cases and TODO apparently some non-error too.
        cached_ast !== undefined
      ) {
        cached_ast.numbered = ast.numbered
      }
    }

    if (context.options.log['ast-pp-simple']) {
      console.error('ast-pp-simple: after pass 4');
      console.error(ast_toplevel.toString());
      console.error();
    }
  }

  context.in_parse = false
  perf_print(context, 'parse_end')
  return ast_toplevel;
}

// Maybe add a paragraph after a \n\n.
function parse_add_paragraph(
  state, ast, new_arg, arg, paragraph_start, paragraph_end, options
) {
  parse_log_debug(state, 'function: parse_add_paragraph');
  parse_log_debug(state, 'paragraph_start: ' + paragraph_start);
  parse_log_debug(state, 'paragraph_end: ' + paragraph_end);
  parse_log_debug(state);
  if (paragraph_start < paragraph_end) {
    const macro = state.macros[arg.get(paragraph_start).macro_name];
    const slice = arg.slice(paragraph_start, paragraph_end);
    if (macro.options.phrasing || slice.length() > 1) {
      // If the first element after the double newline is phrasing content,
      // create a paragraph and put all elements until the next paragraph inside
      // that paragraph.
      new_arg.push(
        new AstNode(
          AstType.MACRO,
          Macro.PARAGRAPH_MACRO_NAME,
          {
            'content': slice
          },
          arg.get(paragraph_start).source_location,
          {
            parent_node: ast,
          }
        )
      );
    } else {
      // Otherwise, don't create the paragraph, and keep all elements as they were.
      new_arg.push(...slice);
    }
  }
}

// Consume one token.
function parse_consume(state) {
  state.i += 1;
  if (state.i < state.tokens.length) {
    state.token = state.tokens[state.i];
  } else {
    throw new Error('programmer error');
  }
  parse_log_debug(state, 'function: parse_consume');
  parse_log_debug(state, 'state.i = ' + state.i.toString())
  parse_log_debug(state, 'state.token = ' + JSON.stringify(state.token));
  parse_log_debug(state);
  return state.token;
}

function parse_log_debug(state, msg='') {
  if (state.options.log.parse) {
    console.error('parse: ' + msg);
  }
}

// Input: e.g. in `\Image[img.jpg]{height=123}` this parses the `[img.jpg]{height=123}`.
// Return value: dict with arguments.
function parse_argument_list(state, macro_name, macro_type) {
  parse_log_debug(state, 'function: parse_argument_list');
  parse_log_debug(state, 'state = ' + JSON.stringify(state.token));
  parse_log_debug(state);
  const args = {};
  const macro = state.macros[macro_name];
  let name_to_arg;
  if (
    // Happens in some error cases.
    macro !== undefined
  ) {
    name_to_arg = macro.name_to_arg
  }
  let positional_arg_count = 0;
  while (
    // End of stream.
    state.token.type !== TokenType.INPUT_END &&
    (
      state.token.type === TokenType.POSITIONAL_ARGUMENT_START ||
      state.token.type === TokenType.NAMED_ARGUMENT_START
    )
  ) {
    let arg_name;
    let open_token = state.token;
    // Consume the *_ARGUMENT_START token out.
    parse_consume(state);
    if (open_token.type === TokenType.POSITIONAL_ARGUMENT_START) {
      if (macro_type === AstType.ERROR) {
        arg_name = positional_arg_count.toString();
      } else {
        if (positional_arg_count >= macro.positional_args.length) {
          parse_error(state,
            `unknown named macro argument "${arg_name}" of macro "${macro_name}"`,
            open_token.source_location,
          );
          arg_name = positional_arg_count.toString();
        } else {
          arg_name = macro.positional_args[positional_arg_count].name;
        }
        positional_arg_count += 1;
      }
    } else {
      // Named argument.
      arg_name = state.token.value;
      if (macro_type !== AstType.ERROR && !(arg_name in macro.named_args)) {
        parse_error(state,
          `unknown named macro argument "${arg_name}" of macro "${macro_name}"`,
          state.token.source_location
        );
      }
      // Parse the argument name out.
      parse_consume(state);
    }
    const arg_children = parse_argument(state, open_token.source_location);
    if (state.token.type !== closing_token(open_token.type)) {
      parse_error(state, `unclosed argument "${open_token.value}"`, open_token.source_location);
    }
    if (
      // Happens in some error cases, e.g. \\undefinedMacro[aa]
      macro !== undefined
    ) {
      const macro_arg = name_to_arg[arg_name];
      const multiple = macro_arg !== undefined && macro_arg.multiple
      if (arg_name in args) {
        if (!multiple) {
          // https://github.com/cirosantilli/cirodown/issues/101
          parse_error(state,
            `named argument "${arg_name}" given multiple times`,
            open_token.source_location,
          )
        }
      } else {
        if (multiple) {
          args[arg_name] = new AstArgument([], open_token.source_location)
        } else {
          args[arg_name] = arg_children;
        }
      }
      if (multiple) {
        args[arg_name].push(new AstNode(
          AstType.MACRO,
          'Comment',
          { 'content': arg_children },
          open_token.source_location,
        ))
      }
    }
    if (state.token.type !== TokenType.INPUT_END) {
      // Consume the *_ARGUMENT_END token out.
      parse_consume(state);
    }
  }
  return args;
}

/**
 * Input: e.g. in `\Image[img.jpg]{height=123}` this parses the `img.jpg` and the `123`.
 * @return AstArgument
 */
function parse_argument(state, open_argument_source_location) {
  const arg_children = new AstArgument([], open_argument_source_location);
  while (
    state.token.type !== TokenType.INPUT_END &&
    state.token.type !== TokenType.POSITIONAL_ARGUMENT_END &&
    state.token.type !== TokenType.NAMED_ARGUMENT_END
  ) {
    // The recursive case: the argument is a lists of macros, go into all of them.
    arg_children.push(parse_macro(state));
  }
return arg_children;
}

// Parse one macro. This is the centerpiece of the parsing!
// Input: e.g. in `\Image[img.jpg]{height=123}` this parses the entire string.
function parse_macro(state) {
  parse_log_debug(state, 'function: parse_macro');
  parse_log_debug(state, 'state = ' + JSON.stringify(state.token));
  parse_log_debug(state);
  if (state.token.type === TokenType.MACRO_NAME) {
    const macro_name = state.token.value;
    const macro_source_location = state.token.source_location;
    let macro_type;
    const unknown_macro_message = `unknown macro name: "${macro_name}"`;
    if (macro_name in state.macros) {
      macro_type = AstType.MACRO;
    } else {
      macro_type = AstType.ERROR;
      parse_error(state, unknown_macro_message);
    }
    // Consume the MACRO_NAME token out.
    parse_consume(state);
    const args = parse_argument_list(state, macro_name, macro_type);
    if (macro_type === AstType.ERROR) {
      return new AstNode(
        macro_type,
        Macro.PLAINTEXT_MACRO_NAME,
        {},
        state.token.source_location,
        {text: error_message_in_output(unknown_macro_message)},
      );
    } else {
      return new AstNode(macro_type, macro_name, args, macro_source_location);
    }
  } else if (state.token.type === TokenType.PLAINTEXT) {
    // Non-recursive case.
    let node = new PlaintextAstNode(
      state.token.value,
      state.token.source_location,
    );
    // Consume the PLAINTEXT node out.
    parse_consume(state);
    return node;
  } else if (state.token.type === TokenType.PARAGRAPH) {
    let node = new AstNode(
      AstType.PARAGRAPH,
      undefined,
      undefined,
      state.token.source_location,
    );
    // Consume the PLAINTEXT node out.
    parse_consume(state);
    return node;
  } else {
    let error_message
    if (
      state.token.type === TokenType.POSITIONAL_ARGUMENT_START ||
      state.token.type === TokenType.NAMED_ARGUMENT_START
    ) {
      error_message = `stray open argument character: '${state.token.value}', maybe you want to escape it with '\\'`;
    } else {
      // Generic error message.
      error_message = `unexpected token ${state.token.type.toString()}`;
    }
    parse_error(state, error_message);
    let node = new PlaintextAstNode(
      error_message_in_output(error_message),
      state.token.source_location,
    );
    // Consume past whatever happened to avoid an infinite loop.
    parse_consume(state);
    return node;
  }
  state.i += 1;
}

function parse_error(state, message, source_location) {
  let new_source_location;
  if (source_location === undefined) {
    new_source_location = new SourceLocation();
  } else {
    new_source_location = source_location.clone();
  }
  new_source_location.path = state.options.input_path;
  if (new_source_location.line === undefined)
    new_source_location.line = state.token.source_location.line;
  if (new_source_location.column === undefined)
    new_source_location.column = state.token.source_location.column;
  state.extra_returns.errors.push(new ErrorMessage(
    message, new_source_location));
}

function path_join(dirname, basename, sep) {
  let ret = dirname;
  if (ret !== '') {
    ret += sep;
  }
  return ret + basename;
}

function path_split(str, sep) {
  const dir_sep_index = str.lastIndexOf(sep)
  if (dir_sep_index == -1) {
    return ['', str];
  } else {
    return [str.substring(0, dir_sep_index), str.substr(dir_sep_index + 1)];
  }
}

function path_splitext(str) {
  const sep_index = str.lastIndexOf('.')
  if (sep_index == -1) {
    return [str, ''];
  } else {
    return [str.substring(0, sep_index), str.substr(sep_index + 1)];
  }
}

function protocol_get(url) {
  const match = /^([a-zA-Z]+):\/\//.exec(url)
  if (match) {
    return match[0]
  } else {
    return null
  }
}

function protocol_is_given(url) {
  return protocol_get(url) !== null
}

function protocol_is_known(src) {
  for (const known_url_protocol of KNOWN_URL_PROTOCOLS) {
    if (src.startsWith(known_url_protocol)) {
      return true;
    }
  }
  return false;
}

// https://cirosantilli.com/cirodown#scope
function remove_toplevel_scope(id, toplevel_ast, context) {
  if (
    toplevel_ast !== undefined &&
    id === toplevel_ast.id
  ) {
    if (toplevel_ast.scope !== undefined) {
      return id.substr(toplevel_ast.scope.length + 1);
    }
    return id;
  } else {
    return id.substr(calculate_scope_length(toplevel_ast));
  }
}

// https://cirosantilli.com/cirodown#index-files
const INDEX_BASENAME_NOEXT = 'index';
exports.INDEX_BASENAME_NOEXT = INDEX_BASENAME_NOEXT;
const INDEX_FILE_BASENAMES_NOEXT = new Set([
  'README',
  INDEX_BASENAME_NOEXT,
]);
exports.INDEX_FILE_BASENAMES_NOEXT = INDEX_FILE_BASENAMES_NOEXT;
const IO_RENAME_MAP = {};
for (let i of INDEX_FILE_BASENAMES_NOEXT) {
  IO_RENAME_MAP[i] = INDEX_BASENAME_NOEXT;
}
exports.IO_RENAME_MAP = IO_RENAME_MAP;
function rename_basename(original) {
  if (original in IO_RENAME_MAP) {
    return IO_RENAME_MAP[original];
  } else {
    return original;
  }
}

function render_error(context, message, source_location, severity=1) {
  if (!context.ignore_errors) {
    context.errors.push(new ErrorMessage(message, source_location, severity));
  }
}

function perf_print(context, name) {
  // Includes and CirodowExample also call convert to parse.
  // For now we are ignoring those recursions. A more correct approach
  // would be to track the time intervals of those subconverts, and add
  // them up to the corresponding toplevel convert.
  if (!context.options.from_include) {
    const now = globals.performance.now();
    const delta = now - context.perf_prev
    context.extra_returns.debug_perf[name] = now
    context.perf_prev = now
    if (context.options.log.perf) {
      console.error(`perf ${name} t=${now} dt=${delta}`);
    }
  }
}
exports.perf_print = perf_print

function propagate_numbered(ast, context) {
  // numbered propagation to children.
  // Note that the property only affects descendants, but not the node itself.
  const parent_tree_node = ast.header_tree_node.parent_node
  if (
    parent_tree_node === undefined ||
    parent_tree_node.ast === undefined
  ) {
    // Try getting parents from \Include.
    // https://github.com/cirosantilli/cirodown/issues/188
    //const parent_asts = ast.get_header_parent_asts(context)
    //if (parent_asts.length > 0) {
    //  ast.numbered = parent_asts.some(ast => ast.numbered)
    //} else {

    ast.numbered = context.options.cirodown_json.h.numbered
  } else {
    const parent_ast = parent_tree_node.ast
    if (parent_ast.validation_output.numbered.given) {
      ast.numbered = parent_ast.validation_output.numbered.boolean
    } else {
      ast.numbered = parent_ast.numbered
    }
  }
}
exports.propagate_numbered = propagate_numbered

// Push all possible IDs due to walking up scope resolution
// into the given ids array.
//
// This very slightly duplicates the resolution code in IdProvider.get,
// but it was not trivial to factor them out, so just going for this now.
function push_scope_resolution(ids, current_scope, id) {
  if (id[0] === Macro.HEADER_SCOPE_SEPARATOR) {
    ids.push(id.substr(1))
  } else {
    if (current_scope !== undefined) {
      current_scope += Macro.HEADER_SCOPE_SEPARATOR
      for (let i = current_scope.length - 1; i > 0; i--) {
        if (current_scope[i] === Macro.HEADER_SCOPE_SEPARATOR) {
          ids.push(current_scope.substring(0, i + 1) + id)
        }
      }
    }
    ids.push(id)
  }
}

// Fuck JavaScript? Can't find a built-in way to get the symbol string without the "Symbol(" part.
// https://stackoverflow.com/questions/30301728/get-the-description-of-a-es6-symbol
function symbol_to_string(symbol) {
  return symbol.toString().slice(7, -1);
}

function title_to_id(title, options) {
  if (options === undefined) {
    options = {}
  }
  if (options.normalize === undefined) {
    options.normalize = {}
  }
  const new_chars = [];
  for (let c of title) {
    if (
      options.normalize === undefined ||
      options.normalize.latin === undefined ||
      options.normalize.latin
    ) {
      c = normalize_latin_character(c)
    }
    if (
      options.normalize === undefined ||
      options.normalize.punctuation === undefined ||
      options.normalize.punctuation
    ) {
      c = normalize_punctuation_character(c)
    }
    c = c.toLowerCase();
    if (!is_ascii(c) || /[a-z0-9-]/.test(c)) {
      new_chars.push(c);
    } else {
      new_chars.push(ID_SEPARATOR);
    }
  }
  return new_chars.join('')
    .replace(new RegExp(ID_SEPARATOR + '+', 'g'), ID_SEPARATOR)
    .replace(new RegExp('^' + ID_SEPARATOR + '+'), '')
    .replace(new RegExp(ID_SEPARATOR + '+$'), '')
  ;
}
exports.title_to_id = title_to_id;

/** Factored out calculations of the ID that is given to each TOC entry.
 *
 * For after everything broke down due to toplevel scope.
 */
function toc_id(target_id_ast, context) {
  return Macro.TOC_PREFIX + target_id_ast.id;
}

function unconvertible(ast, context) {
  const msg = `macro "${ast.macro_name}" must never render`
  if (context.in_parse) {
    render_error(context, msg, ast.source_location);
    return error_message_in_output(msg, context)
  } else {
    throw new Error(msg);
  }
}

function url_basename(str) {
  return basename(str, URL_SEP);
}

// Do some error checking and setup some stuff like boolean.
// We should likely do this in the AstNode constructor. The reason we didn't
// was likely to not need context at that point, and be nicer to serialization.
function validate_ast(ast, context) {
  if (ast.validated) {
    throw new Error(`ast has already been validated:
${ast.toString()}`)
  } else {
    ast.validated = true
  }
  const macro_name = ast.macro_name;
  const macro = context.macros[macro_name];
  const name_to_arg = macro.name_to_arg;
  // First pass sets defaults or missing arguments.
  for (const argname in name_to_arg) {
    ast.validation_output[argname] = {};
    const macro_arg = name_to_arg[argname];
    if (argname in ast.args) {
      ast.validation_output[argname].given = true;
    } else {
      ast.validation_output[argname].given = false;
      if (macro_arg.mandatory) {
        ast.validation_error = [
          `missing mandatory argument ${argname} of ${ast.macro_name}`,
          ast.source_location,
        ];
      }
      if (macro_arg.default !== undefined) {
        ast.args[argname] = new AstArgument([
          new PlaintextAstNode(macro_arg.default, ast.source_location)]);
      } else if (macro_arg.boolean) {
        ast.args[argname] = new AstArgument([new PlaintextAstNode('0', ast.source_location)]);
      }
    }
  }
  // Second pass processes the values including defaults.
  for (const argname in name_to_arg) {
    const macro_arg = name_to_arg[argname];
    if (argname in ast.args) {
      const arg = ast.args[argname];
      if (macro_arg.boolean) {
        let arg_string;
        if (arg.length() > 0) {
          arg_string = render_arg_noescape(arg, context);
        } else {
          arg_string = '1';
        }
        if (arg_string === '0') {
          ast.validation_output[argname].boolean = false;
        } else if (arg_string === '1') {
          ast.validation_output[argname].boolean = true;
        } else {
          ast.validation_output[argname].boolean = false;
          ast.validation_error = [
            `boolean argument "${argname}" of "${ast.macro_name}" has invalid value: "${arg_string}", only "0" and "1" are allowed`,
            arg.source_location
          ];
          break;
        }
      }
      if (macro_arg.positive_nonzero_integer) {
        const arg_string = render_arg_noescape(arg, context);
        const int_value = parseInt(arg_string);
        ast.validation_output[argname]['positive_nonzero_integer'] = int_value;
        if (!Number.isInteger(int_value) || !(int_value > 0)) {
          ast.validation_error = [
            `argument "${argname}" of macro "${ast.macro_name}" must be a positive non-zero integer, got: "${arg_string}"`,
            arg.source_location
          ];
          break;
        }
      }
    }
  }
  if (!macro.options.xss_safe && !context.options.unsafe_xss && !ast.xss_safe) {
    ast.validation_error = [
      `XSS unsafe macro "${macro_name}" used in safe mode: https://cirosantilli.com/cirodown#unsafe-xss`,
      ast.source_location
    ];
  }
}
exports.validate_ast = validate_ast

function x_child_db_effective_id(target_id, context, ast) {
  const target_id_ast = context.id_provider.get(target_id, context, ast.scope);
  if (
    // Can happen if it is in another files that was not extracted yet.
    target_id_ast === undefined
  ) {
    // Then in that case, we can't do scope resolution. But since we are in another file,
    // we must be using the full scope regardless, so it will be correct.
    return target_id
  } else {
    return target_id_ast.id
  }
}

/**
 * @return {[String, String]} [href, content] pair for the x node.
 */
function x_get_href_content(ast, context) {
  const target_id = render_arg_noescape(ast.args.href, context);
  if (context.options.magic_leading_at && target_id[0] === AT_MENTION_CHAR) {
    return [html_attr('href', WEBSITE_URL + target_id.substr(1)), target_id];
  }
  if (target_id[0] === HASHTAG_CHAR) {
    return [html_attr('href', WEBSITE_URL + 'go/topic/' + target_id.substr(1)), target_id];
  }
  const target_id_ast = context.id_provider.get(target_id, context, ast.scope);

  // href
  let href;
  if (target_id_ast) {
    href = x_href_attr(target_id_ast, context);
  } else {
    let message = `cross reference to unknown id: "${target_id}"`;
    let source_location
    if (ast.args.href) {
      source_location = ast.args.href.source_location
    } else {
      source_location = ast.source_location
    }
    render_error(context, message, source_location, 2);
    return [href, error_message_in_output(message, context)];
  }

  // content
  const content_arg = ast.args.content;
  let content;
  if (content_arg === undefined) {
    // No explicit content given, deduce content from target ID title.
    if (context.x_parents.has(ast)) {
      // Prevent render infinite loops.
      let message = `x with infinite recursion`;
      render_error(context, message, ast.source_location);
      return [href, error_message_in_output(message, context)];
    }
    let x_text_options = {
      caption_prefix_span: false,
      capitalize: ast.validation_output.c.boolean,
      from_x: true,
      quote: true,
      pluralize: ast.validation_output.p.given ? ast.validation_output.p.boolean : undefined,
    };
    if (ast.validation_output.full.given) {
      x_text_options.style_full = ast.validation_output.full.boolean;
    }
    const x_parents_new = new Set(context.x_parents);
    x_parents_new.add(ast);
    content = x_text(target_id_ast, clone_and_set(context, 'x_parents', x_parents_new), x_text_options);
    if (content === ``) {
      let message = `empty cross reference body: "${target_id}"`;
      render_error(context, message, ast.source_location);
      return error_message_in_output(message, context);
    }
  } else {
    // Explicit content given, just use it then.
    content = render_arg(content_arg, context);
  }
  return [href, content, target_id_ast];
}

/** Calculate the href value to a given target AstNode.
 *
 * This takes into account e.g. if the target node is in a different source file:
 * https://cirosantilli.com/cirodown#internal-cross-file-references
 *
 * @param {AstNode} target_id_ast
 * @return {String} the value of href (no quotes) that an \x cross reference to the given target_id_ast
 */
function x_href(target_id_ast, context) {
  const [href_path, fragment] = x_href_parts(target_id_ast, context);
  let ret = href_path;
  if (fragment !== '')
    ret += '#' + fragment;
  return ret;
}

// Get the path to the split header version
//
// to_split_headers is set explicitly when making
// links across split/non-split versions of the output.
//
// Otherwise, link to the same type of output as the current one
// as given in in_split_headers.
//
// This way, to_split_hedears in particular forces the link to be
// to the non-split mode, even if we are in split mode.
//
// Some desired sample outcomes:
//
// id='cirodown'             -> ['',       'index-split']
// id='quick-start'          -> ['',       'quick-start']
// id='not-readme'           -> ['',       'not-readme-split']
// id='h2-in-not-the-readme' -> ['',       'h2-in-not-the-readme']
// id='subdir'               -> ['subdir', 'index-split']
// id='subdir/subdir-h2'     -> ['subdir', 'subdir-h2']
// id='subdir/notindex'      -> ['subdir', 'notindex']
// id='subdir/notindex-h2'   -> ['subdir', 'notindex-h2']
function is_to_split_headers(ast, context) {
  return (context.to_split_headers === undefined && ast.split_default) ||
         (context.to_split_headers !== undefined && context.to_split_headers);
}

/** This is the centerpiece of x href calculation!
 *
 * This code is crap. There are too many cases for my brain to handle.
 * So I just write tests, and hack the code until the tests pass, but
 * I'm not capable of factoring it nicely.
 *
 * @param {AstNode} target_id_ast
 */
function x_href_parts(target_id_ast, context) {
  if (target_id_ast.macro_name === Macro.TOC_MACRO_NAME) {
    // Otherwise, split header ToCs would link to the toplevel source ToC,
    // since split header ToCs are not really properly registered.
    return ['', Macro.TOC_ID];
  }
  let target_id_ast_effective_id
  if (target_id_ast.synonym !== undefined) {
    target_id_ast_effective_id = target_id_ast.synonym
  } else {
    target_id_ast_effective_id = target_id_ast.id
  }
  let to_split_headers = is_to_split_headers(target_id_ast, context);
  // Linking to the toplevel of the current output path.
  let to_current_toplevel =
      // Linkting to the current output file.
      target_id_ast_effective_id === context.toplevel_id &&
      // Also requires outputting to the same type of split/nonsplit
      // as the current one.
      context.in_split_headers === to_split_headers
  ;

  // href_path
  let href_path;
  const target_input_path = target_id_ast.source_location.path;
  if (
    target_id_ast.source_location.path === undefined ||
    context.toplevel_output_path === undefined ||
    (
      // Nosplit header link to a header that renders on the
      // same page.
      !context.in_split_headers &&
      !(
        context.to_split_headers !== undefined &&
        context.to_split_headers
      ) &&
      context.options.include_path_set.has(target_input_path)
    ) ||
    (
      // Split header link to image in current header.
      context.in_split_headers &&
      target_id_ast.macro_name !== Macro.HEADER_MACRO_NAME &&
      target_id_ast.get_header_parent_ids(context).has(context.toplevel_id)
    ) ||
    to_current_toplevel
  ) {
    href_path = '';
  } else {
    const [toplevel_output_path_dirname, toplevel_output_path_basename] =
      path_split(context.toplevel_output_path, context.options.path_sep);
    let [
      full_output_path,
      target_output_path_dirname,
      target_output_path_basename
    ] = output_path_from_ast(
      target_id_ast,
      context,
    );
    if (context.options.remove_leading_at) {
      if (target_output_path_dirname) {
        if (target_output_path_dirname[0] === AT_MENTION_CHAR) {
          target_output_path_dirname = target_output_path_dirname.slice(1)
        }
      } else {
        if (
          target_output_path_basename &&
          target_output_path_basename[0] === AT_MENTION_CHAR
        ) {
          target_output_path_basename = target_output_path_basename.slice(1)
        }
      }
    }
    // The target path is the same as the current path being output.
    if (full_output_path === context.toplevel_output_path) {
      href_path = ''
    } else {
      const href_path_dirname_rel = path.relative(
        toplevel_output_path_dirname, target_output_path_dirname);
      if (
        // Same output path.
        href_path_dirname_rel === '' &&
        target_output_path_basename === toplevel_output_path_basename
      ) {
        target_output_path_basename = '';
      } else {
        if (context.options.html_x_extension) {
          target_output_path_basename += '.' + HTML_EXT;
        } else if (target_output_path_basename === INDEX_BASENAME_NOEXT) {
          if (href_path_dirname_rel === '') {
            target_output_path_basename = '.';
          } else {
            target_output_path_basename = '';
          }
        }
      }
      href_path = path_join(href_path_dirname_rel,
        target_output_path_basename, context.options.path_sep);
    }
  }

  // Fragment
  if (
    !context.in_split_headers &&
    context.options.include_path_set.has(target_input_path) &&
    !(context.to_split_headers !== undefined && context.to_split_headers)
  ) {
    // We are not in split headers, and the output is in the current file.
    // Therefore, don't use the split header target no matter what its
    // splitDefault is, use the non-split one.
    to_split_headers = false;
  }
  let fragment;
  if (
    (
      target_id_ast.macro_name === Macro.HEADER_MACRO_NAME &&
      (
        // Linking to a toplevel ID.
        target_id_ast.first_toplevel_child ||
        // Linking towards a split header not included in the current output.
        to_split_headers
      )
    ) ||
    to_current_toplevel
  ) {
    // An empty href means the beginning of the page.
    fragment = '';
  } else {
    // This is the AST that will show up on the top of the rendered page.
    // that contains target_id_ast. We need to know it for toplevel scope culling.
    let toplevel_ast;
    if (to_split_headers) {
      // We know not a header target, as that would have been caught previously.
      toplevel_ast = target_id_ast.get_header_parent_asts(context)[0];
    } else if (
      // The header was included inline into the current file.
      context.options.include_path_set.has(target_input_path) && !context.in_split_headers
    ) {
      toplevel_ast = context.toplevel_ast;
    } else {
      const file_provider_ret = context.options.file_provider.get(target_input_path);
      if (file_provider_ret) {
        toplevel_ast = context.id_provider.get(file_provider_ret.toplevel_id, context);
      } else {
        // The only way this can happen is if we are in the current file, and it hasn't
        // been added to the file db yet.
        toplevel_ast = context.nosplit_toplevel_ast
      }
    }
    fragment = remove_toplevel_scope(target_id_ast_effective_id, toplevel_ast, context);
  }

  // return
  return [html_escape_attr(href_path), html_escape_attr(fragment)];
}

/* href="" that links to a given node. */
function x_href_attr(target_id_ast, context) {
  return html_attr('href', x_href(target_id_ast, context));
}

/**
 * Calculate the text of a cross reference, or the text
 * that the caption text that cross references can refer to, e.g.
 * "Table 123. My favorite table". Both are done in a single function
 * so that style_full references will show very siimlar to the caption
 * they refer to.
 *
 * @param {Object} options
 * @param {Object} href_prefix rendered string containing the href="..."
 *   part of a link to self to be applied e.g. to <>Figure 1<>, of undefined
 *   if this link should not be given.
 */
function x_text(ast, context, options={}) {
  if (!('caption_prefix_span' in options)) {
    options.caption_prefix_span = true;
  }
  if (!('quote' in options)) {
    options.quote = false;
  }
  if (!('fixed_capitalization' in options)) {
    options.fixed_capitalization = true;
  }
  if (!('href_prefix' in options)) {
    options.href_prefix = undefined;
  }
  if (!('force_separator' in options)) {
    options.force_separator = false;
  }
  if (!('from_x' in options)) {
    options.from_x = false;
  }
  if (!('pluralize' in options)) {
    // true: make plural
    // false: make singular
    // undefined: don't touch it
    options.pluralize = undefined;
  }
  if (!('show_caption_prefix' in options)) {
    options.show_caption_prefix = true;
  }
  const macro = context.macros[ast.macro_name];
  let style_full;
  if ('style_full' in options) {
    style_full = options.style_full;
  } else {
    style_full = macro.options.default_x_style_full;
  }
  let ret = ``;
  let number;
  if (style_full) {
    if (options.href_prefix !== undefined) {
      ret += `<a${options.href_prefix}>`
    }
    if (options.show_caption_prefix) {
      if (options.caption_prefix_span) {
        ret += `<span class="caption-prefix">`;
      }
      ret += `${macro.options.caption_prefix} `;
    }
    if (
      ast.numbered &&
      (
        // When in split headers, numbers are only added to headers that
        // are descendants of the toplevel header, thus matching the current ToC.
        // The numbers don't make much sense for other headers.
        ast.macro_name !== Macro.HEADER_MACRO_NAME ||
        (
          // Possible in case of broken header parent=.
          context.toplevel_ast !== undefined &&
          ast.is_header_local_descendant_of(context.toplevel_ast, context)
        )
      )
    ) {
      number = macro.options.get_number(ast, context);
      if (number !== undefined) {
        ret += number;
      }
    }
    if (options.show_caption_prefix && options.caption_prefix_span) {
      ret += `</span>`;
    }
    if (options.href_prefix !== undefined) {
      ret += `</a>`
    }
  }
  let title_arg = macro.options.get_title_arg(ast, context);
  if (
    (
      (title_arg !== undefined && style_full) ||
      options.force_separator
    ) &&
    number !== undefined
  ) {
    ret += html_escape_context(context, `. `);
  }
  if (
    title_arg !== undefined
  ) {
    if (style_full && options.quote) {
      ret += html_escape_context(context, `"`);
    }
    // https://cirosantilli.com/cirodown#cross-reference-title-inflection
    if (options.from_x) {

      // {c}
      let first_ast = title_arg.get(0);
      if (
        ast.macro_name === Macro.HEADER_MACRO_NAME &&
        !ast.validation_output.c.boolean &&
        !style_full &&
        first_ast.node_type === AstType.PLAINTEXT
      ) {
        // https://stackoverflow.com/questions/41474986/how-to-clone-a-javascript-es6-class-instance
        title_arg = lodash.clone(title_arg)
        title_arg.asts = lodash.clone(title_arg.asts)
        title_arg.set(0, new PlaintextAstNode(first_ast.text, first_ast.source_location))
        let txt = title_arg.get(0).text;
        let first_c = txt[0];
        if (options.capitalize) {
          first_c = first_c.toUpperCase();
        } else {
          first_c = first_c.toLowerCase();
        }
        title_arg.get(0).text = first_c + txt.substring(1);
      }

      // {p}
      let last_ast = title_arg.get(title_arg.length() - 1);
      if (
        options.pluralize !== undefined &&
        !style_full &&
        first_ast.node_type === AstType.PLAINTEXT
      ) {
        title_arg = lodash.clone(title_arg)
        title_arg.asts = lodash.clone(title_arg.asts)
        title_arg.set(title_arg.length() - 1, new PlaintextAstNode(last_ast.text, last_ast.source_location));
        title_arg.get(title_arg.length() - 1).text = pluralize(last_ast.text, options.pluralize ? 2 : 1);
      }
    }
    ret += render_arg(title_arg, context);
    if (style_full) {
      const disambiguate_arg = ast.args[Macro.DISAMBIGUATE_ARGUMENT_NAME];
      const title2_arg = ast.args[Macro.TITLE2_ARGUMENT_NAME];
      const show_disambiguate = (disambiguate_arg !== undefined) && macro.options.show_disambiguate;
      if (
        show_disambiguate ||
        (title2_arg !== undefined && title2_arg.asts.length > 0) ||
        ast.title2s.length > 0
      ) {
        ret += ' (';
        const title2_renders = [];
        if (show_disambiguate) {
          title2_renders.push(render_arg(disambiguate_arg, context));
        }
        if (title2_arg !== undefined) {
          title2_renders.push(render_arg(title2_arg, context));
        }
        for (const title2ast of ast.title2s) {
          title2_renders.push(render_arg(title2ast.args.title, context));
        }
        ret += title2_renders.join(', ');
        ret += ')';
      }
      if (options.quote) {
        ret += html_escape_context(context, `"`);
      }
    }
  }
  return ret;
}

// consts

// Dynamic website stuff.
const AT_MENTION_CHAR = '@';
exports.AT_MENTION_CHAR = AT_MENTION_CHAR;
const HASHTAG_CHAR = '#';
const WEBSITE_URL = 'https://ourbigbook.com/';
const REFS_TABLE_PARENT = 'PARENT';
exports.REFS_TABLE_PARENT = REFS_TABLE_PARENT;
const REFS_TABLE_X = 'X';
exports.REFS_TABLE_X = REFS_TABLE_X;
const REFS_TABLE_X_CHILD = 'X_CHILD';
exports.REFS_TABLE_X_CHILD = REFS_TABLE_X_CHILD;
// https://github.com/cirosantilli/cirodown/issues/198
const REFS_TABLE_X_TITLE_TITLE = 'X_TITLE_TITLE';
exports.REFS_TABLE_X_TITLE_TITLE = REFS_TABLE_X_TITLE_TITLE;
const END_NAMED_ARGUMENT_CHAR = '}';
const END_POSITIONAL_ARGUMENT_CHAR = ']';
const ESCAPE_CHAR = '\\';
const HEADER_PARENT_ERROR_MESSAGE = 'header parent either is a previous ID of a level, a future ID, or an invalid ID: '
const HTML_ASCII_WHITESPACE = new Set([' ', '\r', '\n', '\f', '\t']);
const HTML_EXT = 'html';
exports.HTML_EXT = HTML_EXT;
const ID_SEPARATOR = '-';
const INSANE_LIST_START = '* ';
const INSANE_TD_START = '| ';
const INSANE_TH_START = '|| ';
const INSANE_LIST_INDENT = '  ';
const INSANE_HEADER_CHAR = '=';
exports.INSANE_HEADER_CHAR = INSANE_HEADER_CHAR
const LOG_OPTIONS = new Set([
  'ast-inside',
  'ast-pp-simple',
  'parse',
  'perf',
  'split-headers',
  'tokens-inside',
  'tokenize',
]);
exports.LOG_OPTIONS = LOG_OPTIONS;
const IMAGE_EXTENSIONS = new Set([
  'bmp',
  'gif',
  'jpeg',
  'jpg',
  'png',
  'svg',
  'tiff',
  'webp',
])
const VIDEO_EXTENSIONS = new Set([
  'avi',
  'mkv',
  'mov',
  'mp4',
  'ogv',
  'webm',
])
const OUTPUT_FORMAT_CIRODOWN = 'cirodown';
const OUTPUT_FORMAT_HTML = 'html';
const OUTPUT_FORMAT_ID = 'id';
const TOC_ARROW_HTML = '<div class="arrow"><div></div></div>';
const TOC_HAS_CHILD_CLASS = 'has-child';
const MAGIC_CHAR_ARGS = {
  '$': Macro.MATH_MACRO_NAME,
  '`': Macro.CODE_MACRO_NAME,
}
const NAMED_ARGUMENT_EQUAL_CHAR = '=';
const START_NAMED_ARGUMENT_CHAR = '{';
exports.START_NAMED_ARGUMENT_CHAR = START_NAMED_ARGUMENT_CHAR;
const START_POSITIONAL_ARGUMENT_CHAR = '[';
const ESCAPABLE_CHARS = new Set([
  ESCAPE_CHAR,
  START_POSITIONAL_ARGUMENT_CHAR,
  END_POSITIONAL_ARGUMENT_CHAR,
  START_NAMED_ARGUMENT_CHAR,
  END_NAMED_ARGUMENT_CHAR,
  INSANE_LIST_START[0],
  INSANE_TD_START[0],
]);
const INSANE_LINK_END_CHARS = new Set([
  ' ',
  '\n',
  START_POSITIONAL_ARGUMENT_CHAR,
  START_NAMED_ARGUMENT_CHAR,
  END_POSITIONAL_ARGUMENT_CHAR,
  END_NAMED_ARGUMENT_CHAR,
]);
const INSANE_STARTS_TO_MACRO_NAME = {
  [INSANE_LIST_START]:  Macro.LIST_MACRO_NAME,
  [INSANE_TD_START]: Macro.TD_MACRO_NAME,
  [INSANE_TH_START]: Macro.TH_MACRO_NAME,
};
for (const c in MAGIC_CHAR_ARGS) {
  ESCAPABLE_CHARS.add(c);
}
const AstType = make_enum([
  // An in-output error message.
  'ERROR',
  // The most regular and non-magic nodes.
  // Most nodes are of this type.
  'MACRO',
  // A node that contains only text, and no subnodes.
  'PLAINTEXT',
  // Paragraphs are basically MACRO, but with some special
  // magic because of the double newline madness treatment.
  'PARAGRAPH',
]);
const TokenType = make_enum([
  'INPUT_END',
  'MACRO_NAME',
  'NAMED_ARGUMENT_END',
  'NAMED_ARGUMENT_NAME',
  'NAMED_ARGUMENT_START',
  'PARAGRAPH',
  'PLAINTEXT',
  'POSITIONAL_ARGUMENT_END',
  'POSITIONAL_ARGUMENT_START',
]);
const DEFAULT_MEDIA_HEIGHT = 315;
const IMAGE_INLINE_BLOCK_COMMON_NAMED_ARGUMENTS = [
  new MacroArgument({
    name: 'check',
    boolean: true,
  }),
  new MacroArgument({
    name: 'height',
    default: DEFAULT_MEDIA_HEIGHT.toString(),
    positive_nonzero_integer: true,
  }),
  new MacroArgument({
    name: 'provider',
  }),
  new MacroArgument({
    name: 'width',
    positive_nonzero_integer: true,
  }),
]
const MACRO_IMAGE_VIDEO_NAMED_ARGUMENTS = IMAGE_INLINE_BLOCK_COMMON_NAMED_ARGUMENTS.concat([
  new MacroArgument({
    name: Macro.TITLE_ARGUMENT_NAME,
    count_words: true,
  }),
  new MacroArgument({
    name: 'description',
    count_words: true,
  }),
  new MacroArgument({
    name: 'source',
    elide_link_only: true,
  }),
  new MacroArgument({
    name: 'titleFromSrc',
    boolean: true,
  }),
]);

/**
 * Calculate a bunch of default parameters of the media from smart defaults if not given explicitly
 *
 * @return {Object}
 *         MediaProviderType {MediaProviderType} , e.g. type, src, source.
 */
function macro_image_video_resolve_params(ast, context) {
  let error_message;
  let media_provider_type;
  let src = render_arg_noescape(ast.args.src, context);
  let is_url;

  // Provider explicitly given by user on macro.
  if (ast.validation_output.provider.given) {
    const provider_name = render_arg_noescape(ast.args.provider, context);
    if (MEDIA_PROVIDER_TYPES.has(provider_name)) {
      media_provider_type = provider_name;
    } else {
      error_message = `unknown media provider: "${html_escape_attr(provider_name)}"`;
      render_error(context, error_message, ast.args.provider.source_location);
      media_provider_type = 'unknown';
    }
  }

  // Otherwise, detect the media provider.
  let media_provider_type_detected;
  if (src.match(media_provider_type_wikimedia_re)) {
    media_provider_type_detected = 'wikimedia';
  } else if (src.match(media_provider_type_youtube_re)) {
    media_provider_type_detected = 'youtube';
  } else if (protocol_is_known(src)) {
    // Full URL to a website we know nothing about.
    media_provider_type_detected = 'unknown';
  }

  if (media_provider_type_detected === undefined) {
    if (media_provider_type === undefined) {
      // Relative URL, use the default provider if any.
      media_provider_type = context.media_provider_default[ast.macro_name];
    }
    is_url = false;
  } else {
    if (media_provider_type !== undefined && media_provider_type !== media_provider_type_detected) {
      error_message = `detected media provider type "${media_provider_type_detected}", but user also explicitly gave "${media_provider_type}"`;
      render_error(context, error_message, ast.args.provider.source_location);
    }
    if (media_provider_type === undefined) {
      media_provider_type = media_provider_type_detected;
    }
    is_url = true;
  }

  // Fixup src depending for certain providers.
  if (media_provider_type === 'local') {
    const path = context.options.cirodown_json['media-providers'].local.path;
    if (path !== '') {
      src = path + URL_SEP + src;
    }
  } else if (media_provider_type === 'github') {
    src = `https://raw.githubusercontent.com/${context.options.cirodown_json['media-providers'].github.remote}/master/${src}`;
  }

  return {
    error_message,
    media_provider_type: media_provider_type,
    is_url,
    src,
  }
}

function macro_image_video_resolve_params_with_source(ast, context) {
  const ret = macro_image_video_resolve_params(ast, context);
  ret.source = context.macros[ast.macro_name].options.source_func(
    ast, context, ret.src, ret.media_provider_type, ret.is_url);
  return ret;
}

const MACRO_IMAGE_VIDEO_OPTIONS = {
  caption_number_visible: function (ast, context) {
    return 'description' in ast.args ||
      macro_image_video_resolve_params_with_source(ast, context).source !== '';
  },
  get_title_arg: function(ast, context) {
    // Title given explicitly.
    if (ast.validation_output[Macro.TITLE_ARGUMENT_NAME].given) {
      return ast.args[Macro.TITLE_ARGUMENT_NAME];
    }

    // Title from src.
    const media_provider_type = (macro_image_video_resolve_params(ast, context)).media_provider_type;
    if (
      ast.validation_output.titleFromSrc.boolean ||
      (
        !ast.validation_output.titleFromSrc.given &&
        context.options.cirodown_json['media-providers'][media_provider_type]['title-from-src']
      )
    ) {
      let basename_str;
      let src = render_arg(ast.args.src, context);
      if (media_provider_type === 'local') {
        basename_str = url_basename(src);
      } else if (media_provider_type === 'wikimedia') {
        basename_str = context.macros[ast.macro_name].options.image_video_basename(src);
      } else {
        basename_str = src;
      }
      let title_str = basename_str.replace(/_/g, ' ').replace(/\.[^.]+$/, '') + '.';
      return new AstArgument([new PlaintextAstNode(
        title_str, ast.source_location)], ast.source_location);
    }

    // We can't automatically generate one at all.
    return undefined;
  }
}
const MACRO_IMAGE_VIDEO_POSITIONAL_ARGUMENTS = [
  new MacroArgument({
    name: 'src',
    elide_link_only: true,
    mandatory: true,
  }),
  new MacroArgument({
    name: 'alt',
  }),
];
// https://cirosantilli.com/cirodown#known-url-protocols
const KNOWN_URL_PROTOCOL_NAMES = ['http', 'https'];

const KNOWN_URL_PROTOCOLS = new Set()
for (const name of KNOWN_URL_PROTOCOL_NAMES) {
  KNOWN_URL_PROTOCOLS.add(name + '://');
}
const URL_SEP = '/';
exports.URL_SEP = URL_SEP;
const MACRO_WITH_MEDIA_PROVIDER = new Set(['image', 'video']);
const DEFAULT_MACRO_LIST = [
  new Macro(
    Macro.LINK_MACRO_NAME,
    [
      new MacroArgument({
        name: 'href',
        elide_link_only: true,
        mandatory: true,
      }),
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    function(ast, context) {
      let [href, content] = link_get_href_content(ast, context);
      if (ast.validation_output.ref.boolean) {
        content = '<sup class="ref">[ref]</sup>';
      }
      const check = ast.validation_output.check.given ? ast.validation_output.check.boolean : undefined
      const relative = ast.validation_output.relative.given ? ast.validation_output.relative.boolean : undefined
      const attrs = html_render_attrs_id(ast, context);
      return get_link_html({
        attrs,
        check,
        content,
        context,
        href,
        relative,
        source_location: ast.args.href.source_location,
      })
    },
    {
      named_args: [
        new MacroArgument({
          name: 'check',
          boolean: true,
          default: '1',
        }),
        new MacroArgument({
          name: 'relative',
          boolean: true,
        }),
        new MacroArgument({
          name: 'ref',
          boolean: true,
        }),
      ],
      phrasing: true,
    }
  ),
  new Macro(
    'b',
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    html_render_simple_elem(
      'b',
    ),
    {
      phrasing: true,
    }
  ),
  new Macro(
    'br',
    [],
    function(ast, context) {
      return '<br>'
    },
    {
      phrasing: true,
    }
  ),
  new Macro(
    // Block code.
    Macro.CODE_MACRO_NAME.toUpperCase(),
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    function(ast, context) {
      let attrs = html_render_attrs_id(ast, context);
      let content = render_arg(ast.args.content, context);
      let { description, force_separator, multiline_caption } = get_description(ast.args.description, context)
      let ret = `<div class="code${multiline_caption}"${attrs}>\n`;
      if (ast.index_id || ast.validation_output.description.given) {
        ret += `\n<div class="caption">${
          x_text(ast, context, {
            href_prefix: html_self_link(ast, context),
            force_separator
          })}${description}</div>\n`;
      }
      ret += html_code(content);
      ret += `</div>`;
      return ret;
    },
    {
      caption_number_visible: function (ast, context) {
        return 'description' in ast.args
      },
      caption_prefix: 'Code',
      id_prefix: 'code',
      named_args: [
        new MacroArgument({
          name: Macro.TITLE_ARGUMENT_NAME,
          count_words: true,
        }),
        new MacroArgument({
          name: 'description',
          count_words: true,
        }),
      ],
    },
  ),
  new Macro(
    // Inline code.
    Macro.CODE_MACRO_NAME,
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    html_render_simple_elem('code', {newline_after_close: false}),
    {
      phrasing: true,
    }
  ),
  new Macro(
    Macro.CIRODOWN_EXAMPLE_MACRO_NAME,
    [
      new MacroArgument({
        name: 'content',
      }),
    ],
    unconvertible,
    {
      macro_counts_ignore: function(ast) { return true; }
    }
  ),
  new Macro(
    'Comment',
    [
      new MacroArgument({
        name: 'content',
      }),
    ],
    function(ast, context) {
      return '';
    },
    {
      macro_counts_ignore: function(ast) { return true; }
    }
  ),
  new Macro(
    'comment',
    [
      new MacroArgument({
        name: 'content',
      }),
    ],
    function(ast, context) {
      return '';
    },
    {
      phrasing: true,
    }
  ),
  new Macro(
    Macro.HEADER_MACRO_NAME,
    [
      new MacroArgument({
        name: 'level',
        mandatory: true,
        positive_nonzero_integer: true,
      }),
      new MacroArgument({
        name: Macro.TITLE_ARGUMENT_NAME,
        count_words: true,
      }),
    ],
    function(ast, context) {
      if (context.in_header) {
        // Previously was doing an infinite loop when rendering the parent header.
        // But not valid HTML, so I don't think it is worth allowing at all:
        // https://stackoverflow.com/questions/17363465/is-nesting-a-h2-tag-inside-another-header-with-h1-tag-semantically-wrong/71130770#71130770
        const message = `cannot have a header inside another`;
        render_error(context, message, ast.source_location);
        return error_message_in_output(message, context);
      }
      context = clone_and_set(context, 'in_header', true)
      const children = ast.args[Macro.HEADER_CHILD_ARGNAME]
      const tags = ast.args[Macro.HEADER_TAG_ARGNAME]
      if (ast.validation_output.synonym.boolean) {
        if (children !== undefined) {
          const message = `"synonym" and "child" are incompatible`;
          render_error(context, message, children.source_location);
          return error_message_in_output(message, context);
        }
        if (tags !== undefined) {
          const message = `"synonym" and "tags" are incompatible`;
          render_error(context, message, tags.source_location);
          return error_message_in_output(message, context);
        }
        return '';
      }
      let level_int = ast.header_tree_node.get_level();
      if (typeof level_int !== 'number') {
        throw new Error('header level is not an integer after validation');
      }
      let custom_args;
      const level_int_output = level_int - context.header_tree_top_level + 1;
      const is_top_level = level_int === context.header_tree_top_level
      let level_int_capped;
      if (level_int_output > 6) {
        custom_args = {'data-level': new AstArgument([new PlaintextAstNode(
          level_int_output.toString(), ast.source_location)], ast.source_location)};
        level_int_capped = 6;
      } else {
        custom_args = {};
        level_int_capped = level_int_output;
      }
      let attrs = html_render_attrs(ast, context, [], custom_args);
      let id_attr = html_render_attrs_id(ast, context);
      let ret = '';
      // Div that contains h + on hover span.
      ret += `<div class="h"${id_attr}>`;
      ret += `<h${level_int_capped}${attrs}><a${html_self_link(ast, context)} title="link to this element">`;
      let x_text_options = {
        show_caption_prefix: false,
        style_full: true,
      };
      ret += x_text(ast, context, x_text_options);
      ret += `</a>`;
      ret += `</h${level_int_capped}>\n`;

      // On hover metadata.
      let link_to_split;
      let parent_links;
      {
        ret += `<span class="hover-meta"> `;
        if (context.options.split_headers) {
          link_to_split = link_to_split_opposite(ast, context);
          ret += `${HEADER_MENU_ITEM_SEP}${link_to_split}`;
        }
        let toc_href;
        if (!is_top_level && context.has_toc) {
          toc_href = html_attr('href', '#' + toc_id(ast, context));
          ret += `${HEADER_MENU_ITEM_SEP}<a${toc_href} class="cirodown-h-to-toc"${html_attr('title', 'ToC entry for this header')}>${TOC_MARKER}</a>`;
        }

        // Parent links.
        let parent_asts = ast.get_header_parent_asts(context)
        parent_links = [];
        for (const parent_ast of parent_asts) {
          let parent_href = x_href_attr(parent_ast, context);
          let parent_body = render_arg(parent_ast.args[Macro.TITLE_ARGUMENT_NAME], context);
          parent_links.push(`<a${parent_href}${html_attr('title', 'parent header')}>${PARENT_MARKER} "${parent_body}"</a>`);
        }
        parent_links = parent_links.join(HEADER_MENU_ITEM_SEP);
        if (parent_links) {
          ret += `${HEADER_MENU_ITEM_SEP}${parent_links}`;
        }
        let descendant_count = get_descendant_count_html(context, ast.header_tree_node, { long_style: true });
        if (descendant_count !== undefined) {
          ret += `${HEADER_MENU_ITEM_SEP}${descendant_count}`;
        }

        ret += `</span>`;
      }
      ret += `</div>`;

      // Metadata that shows on separate lines below toplevel header.
      let wiki_link;
      if (ast.validation_output.wiki.given) {
        let wiki = render_arg(ast.args.wiki, context);
        if (wiki === '') {
          wiki = (render_arg(ast.args[Macro.TITLE_ARGUMENT_NAME], context)).replace(/ /g, '_');
          if (ast.validation_output[Macro.DISAMBIGUATE_ARGUMENT_NAME].given) {
            wiki += '_(' + (render_arg(ast.args[Macro.DISAMBIGUATE_ARGUMENT_NAME], context)).replace(/ /g, '_')  + ')'
          }
        }
        wiki_link = `<a href="https://en.wikipedia.org/wiki/${html_escape_attr(wiki)}"><span class="fa-brands-400" title="Wikipedia">\u{f266}</span> wiki</a>`;
      }

      // Calculate file_link_html
      let file_link_html
      if (ast.file) {
        file_link_html = get_link_html({
          content: 'View file',
          context,
          href: ast.file,
          relative: undefined,
          source_location: ast.source_location,
        })
      }

      // Calculate tag_ids_html
      const tag_ids = context.id_provider.get_refs_to_as_ids(
        REFS_TABLE_X_CHILD, ast.id);
      const new_context = clone_and_set(context, 'validate_ast', true);
      new_context.source_location = ast.source_location;
      // This is needed because in case of an an undefined \\x with {parent},
      // the undefined target would render as a link on the parent, leading
      // to an error that happens on the header, which is before the actual
      // root cause.
      new_context.ignore_errors = true;
      const tag_ids_html_array = [];
      for (const target_id of Array.from(tag_ids).sort()) {
        const x_ast = new AstNode(
          AstType.MACRO,
          Macro.X_MACRO_NAME,
          {
            'href': new AstArgument(
              [
                new PlaintextAstNode(target_id),
              ],
            ),
            'c': new AstArgument(),
          },
        );
        tag_ids_html_array.push(x_ast.render(new_context));
      }
      const tag_ids_html = `<span title="tags" class="fa-solid-900">\u{f02c}</span> tags: ` + tag_ids_html_array.join(', ');

      // Calculate header_meta and header_meta2
      let header_meta = [];
      let header_meta2 = [];
      let first_header = (
        // May fail in some error scenarios.
        context.toplevel_ast !== undefined &&
        ast.id === context.toplevel_ast.id
      )
      if (file_link_html !== undefined) {
        header_meta.push(file_link_html);
      }
      if (wiki_link !== undefined) {
        header_meta.push(wiki_link);
      }
      if (tag_ids_html_array.length) {
        header_meta.push(tag_ids_html);
      }
      if (first_header) {
        if (parent_links !== '') {
          header_meta2.push(parent_links);
        }
        if (link_to_split !== undefined) {
          header_meta2.push(link_to_split);
        }
        if (context.has_toc) {
          header_meta2.push(`<a${html_attr('href', '#' + Macro.TOC_ID)}>${TOC_MARKER}</a>`);
        }
        let descendant_count_html = get_descendant_count_html(context, ast.header_tree_node, { long_style: true });
        if (descendant_count_html !== undefined) {
          header_meta2.push(descendant_count_html);
        }
      }

      const header_has_meta = header_meta.length > 0 || header_meta2.length > 0
      if (header_has_meta) {
        ret += `<nav class="h-nav h-nav-toplevel">`;
      }
      for (const meta of [header_meta, header_meta2]) {
        if (meta.length > 0) {
          ret += `<div class="nav"> ${meta.join(HEADER_MENU_ITEM_SEP)}</div>`;
        }
      }
      if (header_has_meta) {
        ret += `</nav>\n`;
      }
      if (children !== undefined) {
        ret += header_check_child_tag_exists(ast, context, children, 'child')
      }
      if (tags !== undefined) {
        ret += header_check_child_tag_exists(ast, context, tags, 'tag')
      }
      return ret;
    },
    {
      caption_prefix: 'Section',
      default_x_style_full: false,
      get_number: function(ast, context) {
        let header_tree_node = ast.header_tree_node;
        if (header_tree_node === undefined) {
          return undefined;
        } else {
          return header_tree_node.get_nested_number(context.header_tree_top_level);
        }
      },
      show_disambiguate: true,
      id_prefix: '',
      named_args: [
        new MacroArgument({
          name: 'c',
          boolean: true,
        }),
        new MacroArgument({
          name: Macro.HEADER_CHILD_ARGNAME,
          multiple: true,
        }),
        new MacroArgument({
          name: 'file',
        }),
        new MacroArgument({
          name: 'numbered',
          boolean: true,
        }),
        new MacroArgument({
          name: 'parent',
        }),
        new MacroArgument({
          name: 'scope',
          boolean: true,
        }),
        new MacroArgument({
          name: 'splitDefault',
          boolean: true,
        }),
        new MacroArgument({
          name: 'splitSuffix',
        }),
        new MacroArgument({
          name: Macro.SYNONYM_ARGUMENT_NAME,
          boolean: true,
        }),
        new MacroArgument({
          name: Macro.HEADER_TAG_ARGNAME,
          multiple: true,
        }),
        new MacroArgument({
          name: Macro.TITLE2_ARGUMENT_NAME,
        }),
        // Should I?
        //new MacroArgument({
        //  name: 'tutorial',
        //}),
        new MacroArgument({
          name: 'wiki',
        }),
      ],
    }
  ),
  new Macro(
    'Iframe',
    [
      new MacroArgument({
        name: 'src',
      }),
    ],
    function(ast, context) {
      const attrs = html_render_attrs_id(ast, context, ['src']);
      return `<iframe${attrs}></iframe>`
    },
  ),
  new Macro(
    Macro.INCLUDE_MACRO_NAME,
    [
      new MacroArgument({
        name: 'href',
        mandatory: true,
      }),
    ],
    unconvertible,
    {
      macro_counts_ignore: function(ast) { return true; },
      named_args: [
        new MacroArgument({
          name: 'parent',
        }),
      ],
    }
  ),
  new Macro(
    Macro.LIST_MACRO_NAME,
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    html_render_simple_elem('li', {newline_after_close: true}),
    {
      auto_parent: 'Ul',
      auto_parent_skip: new Set(['Ol']),
    }
  ),
  new Macro(
    // Block math.
    Macro.MATH_MACRO_NAME.toUpperCase(),
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    function(ast, context) {
      let attrs = html_render_attrs_id(ast, context);
      let katex_output = html_katex_convert(ast, context);
      let ret = ``;
      if (ast.validation_output.show.boolean) {
        let href = html_attr('href', '#' + html_escape_attr(ast.id));
        ret += `<div class="math-container"${attrs}>`;
        if (Macro.TITLE_ARGUMENT_NAME in ast.args) {
          ret += `<div class="math-caption-container">\n`;
          ret += `<span class="math-caption">${x_text(ast, context, {href_prefix: href})}</span>`;
          ret += `</div>\n`;
        }
        ret += `<div class="math-equation">\n`
        ret += `<div>${katex_output}</div>\n`;
        ret += `<div><a${href}>(${context.macros[ast.macro_name].options.get_number(ast, context)})</a></div>`;
        ret += `</div>\n`;
        ret += `</div>\n`;
      }
      return ret;
    },
    {
      caption_prefix: 'Equation',
      id_prefix: 'equation',
      get_number: function(ast, context) {
        // Override because unlike other elements such as images, equations
        // always get numbers even if not indexed.
        return ast.macro_count;
      },
      macro_counts_ignore: function(ast) {
        return !ast.validation_output.show.boolean;
      },
      named_args: [
        new MacroArgument({
          name: Macro.TITLE_ARGUMENT_NAME,
        }),
        new MacroArgument({
          boolean: true,
          default: '1',
          name: 'show',
        }),
      ],
    }
  ),
  new Macro(
    // Inline math.
    Macro.MATH_MACRO_NAME,
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    function(ast, context) {
      // KaTeX already adds a <span> for us.
      return html_katex_convert(ast, context);
    },
    {
      phrasing: true,
    }
  ),
  new Macro(
    'i',
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    html_render_simple_elem(
      'i',
    ),
    {
      phrasing: true,
    }
  ),
  new Macro(
    'Image',
    MACRO_IMAGE_VIDEO_POSITIONAL_ARGUMENTS,
    macro_image_video_block_convert_function,
    Object.assign(
      {
        caption_prefix: 'Figure',
        image_video_content_func: function (ast, context, src, rendered_attrs, alt, media_provider_type, is_url) {
          const img_html = html_img({ alt, ast, context, rendered_attrs, src })
          return `<a${html_attr('href', src)}>${img_html}</a>\n`;
        },
        named_args: MACRO_IMAGE_VIDEO_NAMED_ARGUMENTS,
        source_func: function (ast, context, src, media_provider_type, is_url) {
          if ('source' in ast.args) {
            return render_arg(ast.args.source, context);
          } else if (media_provider_type == 'wikimedia') {
            return macro_image_video_block_convert_function_wikimedia_source_url +
              context.macros[ast.macro_name].options.image_video_basename(src);
          } else {
            return '';
          }
        }
      },
      Object.assign(
        {
          image_video_basename: function(src) {
            return url_basename(html_escape_attr(src)).replace(
              macro_image_video_block_convert_function_wikimedia_source_image_re, '');
          },
        },
        MACRO_IMAGE_VIDEO_OPTIONS,
      ),
    ),
  ),
  new Macro(
    'image',
    MACRO_IMAGE_VIDEO_POSITIONAL_ARGUMENTS,
    function(ast, context) {
      let alt_arg;
      if (ast.args.alt === undefined) {
        alt_arg = ast.args.src;
      } else {
        alt_arg = ast.args.alt;
      }
      let alt = html_attr('alt', html_escape_attr(render_arg(alt_arg, context)));
      let rendered_attrs = html_render_attrs_id(ast, context, ['height', 'width']);
      let { error_message, src } = macro_image_video_resolve_params(ast, context);
      let ret = html_img({ alt, ast, context, rendered_attrs, src })
      if (error_message) {
        ret += error_message_in_output(error_message, context)
      }
      return ret
    },
    {
      named_args: IMAGE_INLINE_BLOCK_COMMON_NAMED_ARGUMENTS,
      phrasing: true,
    }
  ),
  new Macro(
    'JsCanvasDemo',
    [
      new MacroArgument({
        name: 'content',
        mandatory: true,
      }),
    ],
    function(ast, context) {
      return html_code(
        render_arg(ast.args.content, context),
        {'class': 'cirodown-js-canvas-demo'}
      );
    },
    {
      xss_safe: false,
    }
  ),
  new Macro(
    'Ol',
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
        remove_whitespace_children: true,
      }),
    ],
    html_render_simple_elem('ol', {newline_after_open: true}),
  ),
  new Macro(
    Macro.PARAGRAPH_MACRO_NAME,
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    html_render_simple_elem(
      'div',
      {
        attrs: {'class': 'p'},
      }
    ),
  ),
  new Macro(
    Macro.PLAINTEXT_MACRO_NAME,
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    function(ast, context) {
      return html_escape_context(context, ast.text);
    },
    {
      phrasing: true,
    }
  ),
  new Macro(
    'passthrough',
    [
      new MacroArgument({
        name: 'content',
      }),
    ],
    function(ast, context) {
      return render_arg_noescape(ast.args.content, context);
    },
    {
      phrasing: true,
      xss_safe: false,
    }
  ),
  new Macro(
    'Q',
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    html_render_simple_elem(
      'blockquote',
    ),
    {
      caption_prefix: 'Quotation',
    }
  ),
  new Macro(
    'sub',
    [
      new MacroArgument({
        name: 'content',
      }),
    ],
    html_render_simple_elem(
      'sub',
    ),
    {
      phrasing: true,
    }
  ),
  new Macro(
    'sup',
    [
      new MacroArgument({
        name: 'content',
      }),
    ],
    html_render_simple_elem(
      'sup',
    ),
    {
      phrasing: true,
    }
  ),
  new Macro(
    Macro.TABLE_MACRO_NAME,
    [
      new MacroArgument({
        name: 'content',
        remove_whitespace_children: true,
        count_words: true,
      }),
    ],
    function(ast, context) {
      let attrs = html_render_attrs_id(ast, context);
      let content = render_arg(ast.args.content, context);
      let ret = ``;
      let { description, force_separator, multiline_caption } = get_description(ast.args.description, context)
      ret += `<div class="table${multiline_caption}"${attrs}>\n`;
      // TODO not using caption because I don't know how to allow the caption to be wider than the table.
      // I don't want the caption to wrap to a small table size.
      //
      // If we ever solve that, re-add the following style:
      //
      // caption {
      //   color: black;
      //   text-align: left;
      // }
      //
      //Caption on top as per: https://tex.stackexchange.com/questions/3243/why-should-a-table-caption-be-placed-above-the-table */
      let href = html_attr('href', '#' + html_escape_attr(ast.id));
      if (ast.index_id || ast.validation_output.description.given) {
        ret += `<div class="caption">${x_text(ast, context, {
          href_prefix: href,
          force_separator,
        })}${description}</div>`;
      }
      ret += `<table>\n${content}</table>\n`;
      ret += `</div>\n`;
      return ret;
    },
    {
      caption_number_visible: function (ast, context) {
        return 'description' in ast.args
      },
      named_args: [
        new MacroArgument({
          name: 'description',
          count_words: true,
        }),
        new MacroArgument({
          name: Macro.TITLE_ARGUMENT_NAME,
          count_words: true,
        }),
      ],
    }
  ),
  new Macro(
    Macro.TD_MACRO_NAME,
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    html_render_simple_elem('td', {newline_after_close: true}),
    {
      newline_after_close: true,
    }
  ),
  new Macro(
    Macro.TOC_MACRO_NAME,
    [],
    function(ast, context) {
      // Not rendering ID here because that function does scope culling. But TOC ID is a fixed value without scope for now.
      // so that was removing the TOC id in subdirectories.
      let attrs = html_render_attrs(ast, context);
      let todo_visit = [];
      let top_level = 0;
      let root_node = context.header_tree;
      let ret = `<div id="${Macro.TOC_ID}"class="toc-container"${attrs}>\n<ul>\n<li${html_class_attr([TOC_HAS_CHILD_CLASS, 'toplevel'])}><div class="title-div">`;
      if (root_node.children.length === 1) {
        root_node = root_node.children[0];
      }
      let descendant_count_html = get_descendant_count_html_sep(context, root_node, { long_style: false, show_descendant_count: true });
      ret += `${TOC_ARROW_HTML}<span class="not-arrow"><a class="title"${x_href_attr(ast, context)}>Table of contents</a><span class="hover-metadata">${descendant_count_html}</span></span></div>\n`;
      for (let i = root_node.children.length - 1; i >= 0; i--) {
        todo_visit.push([root_node.children[i], 1]);
      }
      if (todo_visit.length === 0) {
        // Empty ToC. Don't render. Initial common case: leaf split header nodes.
        return '';
      }
      let linear_count = 0
      while (todo_visit.length > 0) {
        const [tree_node, level] = todo_visit.pop();
        if (level > top_level) {
          ret += `<ul>\n`;
        } else if (level < top_level) {
          ret += `</li>\n</ul>\n`.repeat(top_level - level);
        } else {
          ret += `</li>\n`;
        }
        ret += '<li';
        if (tree_node.children.length > 0) {
          ret += html_class_attr([TOC_HAS_CHILD_CLASS]);
        }
        ret += '>'
        let target_id_ast = context.id_provider.get(tree_node.ast.id, context);
        if (
          // Can happen in test error cases:
          // - cross reference from header title without ID to previous header is not allowed
          // - include to file that does exists without embed includes before extracting IDs fails gracefully
          target_id_ast !== undefined
        ) {
          // ToC entries always link to the same split/nosplit type, except for included sources.
          // This might be handled more generally through: https://github.com/cirosantilli/cirodown/issues/146
          // but for now we are just taking care of this specific and important ToC subcase.
          let cur_context;
          if (ast.source_location.path === target_id_ast.source_location.path) {
            cur_context = clone_and_set(context, 'to_split_headers', context.in_split_headers);
          } else {
            cur_context = context;
          }

          let content = x_text(target_id_ast, cur_context, {style_full: true, show_caption_prefix: false});
          let href = x_href_attr(target_id_ast, cur_context);
          const my_toc_id = toc_id(target_id_ast, cur_context);
          let id_to_toc = html_attr(Macro.ID_ARGUMENT_NAME, my_toc_id);
          // The inner <div></div> inside arrow is so that:
          // - outter div: takes up space to make clicking easy
          // - inner div: minimal size to make the CSS arrow work, but too small for confortable clicking
          let descendant_count_html = get_descendant_count_html_sep(context, tree_node, { long_style: false, show_descendant_count: false });
          let linear_count_str
          if (context.options.add_test_instrumentation) {
            linear_count_str = html_attr('data-test', linear_count)
          } else {
            linear_count_str = ''
          }
          ret += `<div${id_to_toc}>${TOC_ARROW_HTML}<span class="not-arrow"><a${href}${linear_count_str}>${content}</a><span class="hover-metadata">`;

          let toc_href = html_attr('href', '#' + my_toc_id);
          ret += `${HEADER_MENU_ITEM_SEP}<a${toc_href}${html_attr('title', 'link to this ToC entry')}>${UNICODE_LINK} link</a>`;
          if (cur_context.options.split_headers) {
            ret += `${HEADER_MENU_ITEM_SEP}${link_to_split_opposite(target_id_ast, cur_context)}`;
          }
          let parent_ast = target_id_ast.get_header_parent_asts(cur_context)[0];
          if (
            // Possible on broken h1 level.
            parent_ast !== undefined
          ) {
            let parent_href_target;
            if (
              parent_ast.header_tree_node !== undefined &&
              parent_ast.header_tree_node.get_level() === cur_context.header_tree_top_level
            ) {
              parent_href_target = Macro.TOC_ID;
            } else {
              parent_href_target = toc_id(parent_ast, cur_context);
            }
            let parent_href = html_attr('href', '#' + parent_href_target);
            let parent_body = render_arg(parent_ast.args[Macro.TITLE_ARGUMENT_NAME], context);
            ret += `${HEADER_MENU_ITEM_SEP}<a${parent_href}${html_attr('title', 'parent ToC entry')}>${PARENT_MARKER} "${parent_body}"</a>`;
          }
          ret += `${descendant_count_html}</span></span></div>`;
          linear_count++
        }
        if (tree_node.children.length > 0) {
          for (let i = tree_node.children.length - 1; i >= 0; i--) {
            todo_visit.push([tree_node.children[i], level + 1]);
          }
          ret += `\n`;
        }
        top_level = level;
      }
      ret += `</li>\n</ul>\n`.repeat(top_level);
      // Close the table of contents list.
      ret += `</li>\n</ul>\n`;
      ret += `</div>\n`
      return ret;
    },
  ),
  new Macro(
    Macro.TOPLEVEL_MACRO_NAME,
    [
      new MacroArgument({
        count_words: true,
        name: 'content',
      }),
    ],
    function(ast, context) {
      let title = ast.args[Macro.TITLE_ARGUMENT_NAME];
      if (title === undefined) {
        let text_title;
        if (Macro.TITLE_ARGUMENT_NAME in context.options) {
          text_title = context.options[Macro.TITLE_ARGUMENT_NAME];
        } else if (context.header_tree.children.length > 0) {
          text_title = render_arg(
            context.header_tree.children[0].ast.args[Macro.TITLE_ARGUMENT_NAME],
            clone_and_set(context, 'id_conversion', true)
          );
        } else {
          text_title = 'dummy title because title is mandatory in HTML';
        }
        title = new AstArgument([
          new PlaintextAstNode(text_title, ast.source_location)],
          ast.source_location,
          text_title
        );
      }
      let ret;
      let body = render_arg(ast.args.content, context);

      // Footer metadata.
      if (context.toplevel_ast !== undefined) {
        {
          const target_ids = context.id_provider.get_refs_to_as_ids(
            REFS_TABLE_X_CHILD, context.toplevel_ast.id, true);
          body += create_link_list(context, ast, 'tagged', 'Tagged', target_ids)
        }

        // Ancestors
        {
          const ancestors = [];
          let cur_ast = context.toplevel_ast;
          while (true) {
            cur_ast = cur_ast.get_header_parent_asts(context)[0];
            if (cur_ast === undefined) {
              break
            }
            ancestors.push(cur_ast);
          }
          if (ancestors.length !== 0) {
            // TODO factor this out more with real headers.
            const id = 'ancestors'
            body += `<div>${html_hide_hover_link('#ancestors')}<h2 id="#${id}"><a href="#ancestors">Ancestors</a></h2></div>\n`;
            const ancestor_id_asts = [];
            for (const ancestor of ancestors) {
              //let counts_str;
              //if (ancestor.header_tree_node !== undefined) {
              //  counts_str = get_descendant_count_html_sep(ancestor.header_tree_node, false);
              //} else {
              //  counts_str = '';
              //}
              ancestor_id_asts.push(new AstNode(
                AstType.MACRO,
                Macro.LIST_MACRO_NAME,
                {
                  'content': new AstArgument(
                    [
                      new AstNode(
                        AstType.MACRO,
                        Macro.X_MACRO_NAME,
                        {
                          'href': new AstArgument(
                            [
                              new PlaintextAstNode(ancestor.id),
                            ],
                          ),
                          'c': new AstArgument(),
                        },
                      ),
                      //new AstNode(
                      //  AstType.MACRO,
                      //  'passthrough',
                      //  {
                      //    'content': new AstArgument(
                      //      [
                      //        new PlaintextAstNode(counts_str),
                      //      ],
                      //    ),
                      //  },
                      //  undefined,
                      //  {
                      //    xss_safe: true,
                      //  }
                      //),
                    ],
                  ),
                },
              ));
            }
            const ulArgs = {
              'content': new AstArgument(ancestor_id_asts)
            }
            if (context.options.add_test_instrumentation) {
              ulArgs[Macro.TEST_DATA_ARGUMENT_NAME] = [new PlaintextAstNode(id)]
            }
            const incoming_ul_ast = new AstNode(
              AstType.MACRO,
              'Ul',
              ulArgs,
            );
            const new_context = clone_and_set(context, 'validate_ast', true);
            new_context.source_location = ast.source_location;
            body += incoming_ul_ast.render(new_context);
          }
        }

        {
          const target_ids = context.id_provider.get_refs_to_as_ids(REFS_TABLE_X, context.toplevel_ast.id);
          body += create_link_list(context, ast, 'incoming-links', 'Incoming links', target_ids)
        }
      }

      if (context.options.body_only) {
        ret = body;
      } else {
        let template;
        if (context.options.template !== undefined) {
          template = context.options.template;
        } else {
          template = `<!doctype html>
<html lang=en>
<head>
<meta charset=utf-8>
<title>{{ title }}</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>{{ style }}</style>
{{ head }}</head>
<body class="cirodown">
{{ body }}
{{ post_body }}</body>
</html>
`;
        }

        let root_page;
        if (context.options.html_x_extension) {
          context.options.template_vars.html_ext = '.html';
          root_page = context.options.template_vars.root_relpath + INDEX_BASENAME_NOEXT + '.' + HTML_EXT;
        } else {
          context.options.template_vars.html_ext = '';
          if (context.options.template_vars.root_relpath === '') {
            root_page = '.'
          } else {
            root_page = context.options.template_vars.root_relpath;
          }
        }
        if (root_page === context.toplevel_output_path) {
          root_page = '';
        }
        const render_env = {
          body: body,
          root_page: root_page,
          title: render_arg(title, context),
        };
        Object.assign(render_env, context.options.template_vars);

        // Resolve relative styles and scripts.
        let relative_scripts = [];
        for (const script of context.options.template_scripts_relative) {
          relative_scripts.push(`<script src="${context.options.template_vars.root_relpath}${script}"></script>\n`);
        }
        render_env.post_body = relative_scripts.join('') + render_env.post_body + "<script>cirodown_runtime.cirodown_runtime()</script>\n";
        let relative_styles = [];
        for (const style of context.options.template_styles_relative) {
          relative_styles.push(`@import "${context.options.template_vars.root_relpath}${style}";\n`);
        }
        render_env.style = relative_styles.join('') + render_env.style;

        const { Liquid } = require('liquidjs');
        ret = (new Liquid()).parseAndRenderSync(
          template,
          render_env,
          {
            strictFilters: true,
            strictVariables: true,
          }
        );
      }
      return ret;
    },
    {
      macro_counts_ignore: function(ast) { return true; },
      named_args: [
        new MacroArgument({
          name: Macro.TITLE_ARGUMENT_NAME,
          count_words: true,
        }),
      ],
    }
  ),
  new Macro(
    Macro.TH_MACRO_NAME,
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    html_render_simple_elem('th', {newline_after_close: true}),
  ),
  new Macro(
    Macro.TR_MACRO_NAME,
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
        remove_whitespace_children: true,
      }),
    ],
    function(ast, context) {
      let content_ast = ast.args.content;
      let content = render_arg(content_ast, context);
      let res = '';
      if (ast.args.content.get(0).macro_name === Macro.TH_MACRO_NAME) {
        if (
          ast.parent_argument_index === 0 ||
          ast.parent_argument.get(ast.parent_argument_index - 1).args.content.get(0).macro_name !== Macro.TH_MACRO_NAME
        ) {
          res += `<thead>\n`;
        }
      }
      if (ast.args.content.get(0).macro_name === Macro.TD_MACRO_NAME) {
        if (
          ast.parent_argument_index === 0 ||
          ast.parent_argument.get(ast.parent_argument_index - 1).args.content.get(0).macro_name !== Macro.TD_MACRO_NAME
        ) {
          res += `<tbody>\n`;
        }
      }
      res += `<tr${html_render_attrs_id(ast, context)}>\n${content}</tr>\n`;
      if (ast.args.content.get(0).macro_name === Macro.TH_MACRO_NAME) {
        if (
          ast.parent_argument_index === ast.parent_argument.length() - 1 ||
          ast.parent_argument.get(ast.parent_argument_index + 1).args.content.get(0).macro_name !== Macro.TH_MACRO_NAME
        ) {
          res += `</thead>\n`;
        }
      }
      if (ast.args.content.get(0).macro_name === Macro.TD_MACRO_NAME) {
        if (
          ast.parent_argument_index === ast.parent_argument.length() - 1 ||
          ast.parent_argument.get(ast.parent_argument_index + 1).args.content.get(0).macro_name !== Macro.TD_MACRO_NAME
        ) {
          res += `</tbody>\n`;
        }
      }
      return res;
    },
    {
      auto_parent: Macro.TABLE_MACRO_NAME,
    }
  ),
  new Macro(
    'Ul',
    [
      new MacroArgument({
        name: 'content',
        count_words: true,
        remove_whitespace_children: true,
      }),
    ],
    html_render_simple_elem('ul', {
      newline_after_open: true,
      wrap: true,
    }),
  ),
  new Macro(
    Macro.X_MACRO_NAME,
    [
      new MacroArgument({
        name: 'href',
        mandatory: true,
      }),
      new MacroArgument({
        name: 'content',
        count_words: true,
      }),
    ],
    function(ast, context) {
      let [href, content, target_ast] = x_get_href_content(ast, context);
      if (
        ast.validation_output.full.given &&
        ast.validation_output.ref.given
      ) {
        const message = `"full" and "ref" are incompatible`;
        render_error(context, message, ast.source_location);
        content = error_message_in_output(message, context);
      } else if (ast.validation_output.ref.boolean) {
        content = '*';
      }
      if (context.x_parents.size === 0) {
        // Counts.
        let counts_str;
        if (
          // Happens on error case of linking to non existent ID.
          target_ast === undefined ||
          // Happens for cross links. TODO make those work too...
          target_ast.parent_node === undefined
        ) {
          counts_str = '';
        } else {
          const counts = get_descendant_count(target_ast.header_tree_node);
          for (let i = 0; i < counts.length; i++) {
            counts[i] = format_number_approx(counts[i]);
          }
          counts_str = `\nword count: ${counts[0]}\ndescendant word count: ${counts[2]}\ndescendant count: ${counts[1]}`;
        }
        const attrs = html_render_attrs_id(ast, context);
        return `<a${href}${attrs}${html_attr('title', 'internal link' + counts_str)}>${content}</a>`;
      } else {
        return content;
      }
    },
    {
      named_args: [
        new MacroArgument({
          name: 'c',
          boolean: true,
        }),
        new MacroArgument({
          // https://github.com/cirosantilli/cirodown/issues/92
          name: 'child',
          boolean: true,
        }),
        new MacroArgument({
          name: 'full',
          boolean: true,
        }),
        new MacroArgument({
          name: 'p',
          boolean: true,
        }),
        new MacroArgument({
          // https://github.com/cirosantilli/cirodown/issues/92
          name: 'parent',
          boolean: true,
        }),
        new MacroArgument({
          name: 'ref',
          boolean: true,
        }),
      ],
      phrasing: true,
    }
  ),
  new Macro(
    'Video',
    MACRO_IMAGE_VIDEO_POSITIONAL_ARGUMENTS,
    macro_image_video_block_convert_function,
    Object.assign(
      {
        caption_prefix: 'Video',
        image_video_basename: function(src) {
          return url_basename(html_escape_attr(src)).replace(
            macro_image_video_block_convert_function_wikimedia_source_video_re, '$1');
        },
        image_video_content_func: function (ast, context, src, rendered_attrs, alt, media_provider_type, is_url) {
          if (media_provider_type === 'youtube') {
            let url_start_time;
            let video_id;
            if (is_url) {
              const url = new URL(src);
              const url_params = url.searchParams;
              if (url_params.has('t')) {
                url_start_time = url_params.get('t');
              }
              if (url.hostname === 'youtube.com' || url.hostname === 'www.youtube.com') {
                if (url_params.has('v')) {
                  video_id = url_params.get('v')
                } else {
                  let message = `youtube URL without video ID "${src}"`;
                  render_error(context, message, ast.source_location);
                  return error_message_in_output(message, context);
                }
              } else {
                // youtu.be/<ID> and path is "/<ID>" so get rid of "/".
                video_id = url.pathname.substr(1);
              }
            } else {
              video_id = src;
            }
            let start_time;
            if ('start' in ast.args) {
              start_time = ast.validation_output.start.positive_nonzero_integer;
            } else if (url_start_time !== undefined) {
              start_time = html_escape_attr(url_start_time);
            }
            let start;
            if (start_time !== undefined) {
              start = `?start=${start_time}`;
            } else {
              start = '';
            }
            return `<iframe width="560" height="${DEFAULT_MEDIA_HEIGHT}" loading="lazy" src="https://www.youtube.com/embed/${html_escape_attr(video_id)}${start}" ` +
                  `allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>`;
          } else {
            const check = ast.validation_output.check.given ? ast.validation_output.check.boolean : undefined
            let error
            ;({ href: src, error } = check_and_update_local_link({
              check: ast.validation_output.check.boolean,
              context,
              href: src,
              source_location: ast.args.src.source_location,
            }))
            let start;
            if ('start' in ast.args) {
              // https://stackoverflow.com/questions/5981427/start-html5-video-at-a-particular-position-when-loading
              start = `#t=${ast.validation_output.start.positive_nonzero_integer}`;
            } else {
              start = '';
            }
            return `<video${html_attr('src', src + start)}${rendered_attrs} preload="none" controls${alt}></video>${error}\n`;
          }
        },
        named_args: MACRO_IMAGE_VIDEO_NAMED_ARGUMENTS.concat(
          new MacroArgument({
            name: 'start',
            positive_nonzero_integer: true,
          }),
        ),
        source_func: function (ast, context, src, media_provider_type, is_url) {
          if ('source' in ast.args) {
            return render_arg(ast.args.source, context);
          } else if (media_provider_type === 'youtube') {
            if (is_url) {
              return html_escape_attr(src);
            } else {
              return `https://youtube.com/watch?v=${html_escape_attr(src)}`;
            }
          } else if (media_provider_type === 'wikimedia') {
            return macro_image_video_block_convert_function_wikimedia_source_url +
              context.macros[ast.macro_name].options.image_video_basename(src);
          } else {
            return '';
          }
        }
      },
      MACRO_IMAGE_VIDEO_OPTIONS,
    ),
  ),
];

function create_link_list(context, ast, id, title, target_ids, body) {
  let ret = '';
  if (target_ids.size !== 0) {
    // TODO factor this out more with real headers.
    const target_id_asts = [];
    ret += `<div>${html_hide_hover_link('#' + id)}<h2 id="${id}"><a href="#${id}">${title}</a></h2></div>\n`;
    for (const target_id of Array.from(target_ids).sort()) {
      let target_ast = context.id_provider.get(target_id, context);
      if (
        // Possible when user sets an invalid ID on \x with child \x[invalid]{child}.
        // The error is caught elsewhere.
        target_ast !== undefined
      ) {
        //let counts_str;
        //if (target_ast.header_tree_node !== undefined) {
        //  counts_str = get_descendant_count_html_sep(target_ast.header_tree_node, false);
        //} else {
        //  counts_str = '';
        //}
        target_id_asts.push(new AstNode(
          AstType.MACRO,
          Macro.LIST_MACRO_NAME,
          {
            'content': new AstArgument(
              [
                new AstNode(
                  AstType.MACRO,
                  Macro.X_MACRO_NAME,
                  {
                    'href': new AstArgument(
                      [
                        new PlaintextAstNode(target_id),
                      ],
                    ),
                    'c': new AstArgument(),
                  },
                ),
                //new AstNode(
                //  AstType.MACRO,
                //  'passthrough',
                //  {
                //    'content': new AstArgument(
                //      [
                //        new PlaintextAstNode(counts_str),
                //      ],
                //    ),
                //  },
                //  undefined,
                //  {
                //    xss_safe: true,
                //  }
                //),
              ],
            ),
          },
        ));
      }
    }
    let ulArgs = {
      'content': new AstArgument(target_id_asts)
    }
    if (context.options.add_test_instrumentation) {
      ulArgs[Macro.TEST_DATA_ARGUMENT_NAME] = [new PlaintextAstNode(id)]
    }
    const incoming_ul_ast = new AstNode(AstType.MACRO, 'Ul', ulArgs)
    const new_context = clone_and_set(context, 'validate_ast', true);
    new_context.source_location = ast.source_location;
    ret += incoming_ul_ast.render(new_context);
  }
  return ret
}

function cirodown_convert_simple_elem(ast, context) {
  return ESCAPE_CHAR +
    ast.macro_name +
    START_POSITIONAL_ARGUMENT_CHAR +
    render_arg(ast.args.content, context) +
    END_POSITIONAL_ARGUMENT_CHAR;
}

const MACRO_CONVERT_FUNCIONS = {
  [OUTPUT_FORMAT_CIRODOWN]: {
    [Macro.LINK_MACRO_NAME]: function(ast, context) {
      const [href, content] = link_get_href_content(ast, context);
      return content;
    },
    'b': cirodown_convert_simple_elem,
    [Macro.CODE_MACRO_NAME.toUpperCase()]: id_convert_simple_elem(),
    [Macro.CODE_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.CIRODOWN_EXAMPLE_MACRO_NAME]: unconvertible,
    'Comment': cirodown_convert_simple_elem,
    'comment': cirodown_convert_simple_elem,
    [Macro.HEADER_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.INCLUDE_MACRO_NAME]: cirodown_convert_simple_elem,
    [Macro.LIST_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.MATH_MACRO_NAME.toUpperCase()]: id_convert_simple_elem(),
    [Macro.MATH_MACRO_NAME]: id_convert_simple_elem(),
    'i': cirodown_convert_simple_elem,
    'Image': function(ast, context) { return ''; },
    'image': function(ast, context) { return ''; },
    'JsCanvasDemo': id_convert_simple_elem(),
    'Ol': cirodown_convert_simple_elem,
    [Macro.PARAGRAPH_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.PLAINTEXT_MACRO_NAME]: function(ast, context) {return ast.text},
    'passthrough': id_convert_simple_elem(),
    'Q': cirodown_convert_simple_elem,
    [Macro.TABLE_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.TD_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.TOC_MACRO_NAME]: function(ast, context) { return '' },
    [Macro.TOPLEVEL_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.TH_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.TR_MACRO_NAME]: id_convert_simple_elem(),
    'Ul': id_convert_simple_elem(),
    [Macro.X_MACRO_NAME]: id_convert_simple_elem('href'),
    'Video': macro_image_video_block_convert_function,
  },
  [OUTPUT_FORMAT_ID]: {
    [Macro.LINK_MACRO_NAME]: function(ast, context) {
      const [href, content] = link_get_href_content(ast, context);
      return content;
    },
    'b': id_convert_simple_elem(),
    [Macro.CODE_MACRO_NAME.toUpperCase()]: id_convert_simple_elem(),
    [Macro.CODE_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.CIRODOWN_EXAMPLE_MACRO_NAME]: unconvertible,
    'Comment': function(ast, context) { return ''; },
    'comment': function(ast, context) { return ''; },
    [Macro.HEADER_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.INCLUDE_MACRO_NAME]: unconvertible,
    [Macro.LIST_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.MATH_MACRO_NAME.toUpperCase()]: id_convert_simple_elem(),
    [Macro.MATH_MACRO_NAME]: id_convert_simple_elem(),
    'i': id_convert_simple_elem(),
    'Image': function(ast, context) { return ''; },
    'image': function(ast, context) { return ''; },
    'JsCanvasDemo': id_convert_simple_elem(),
    'Ol': id_convert_simple_elem(),
    [Macro.PARAGRAPH_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.PLAINTEXT_MACRO_NAME]: function(ast, context) {return ast.text},
    'passthrough': id_convert_simple_elem(),
    'Q': id_convert_simple_elem(),
    [Macro.TABLE_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.TD_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.TOC_MACRO_NAME]: function(ast, context) { return '' },
    [Macro.TOPLEVEL_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.TH_MACRO_NAME]: id_convert_simple_elem(),
    [Macro.TR_MACRO_NAME]: id_convert_simple_elem(),
    'Ul': id_convert_simple_elem(),
    [Macro.X_MACRO_NAME]: function(ast, context) {
      if (ast.args.content) {
        return id_convert_simple_elem('content')(ast, context)
      } else {
        return id_convert_simple_elem('href')(ast, context)
      }
    },
    'Video': macro_image_video_block_convert_function,
  },
};
const TOPLEVEL_CHILD_MODIFIER = {
  [OUTPUT_FORMAT_CIRODOWN]: function(ast, context, out) {
    return out;
  },
  [OUTPUT_FORMAT_HTML]: function(ast, context, out) {
    return `<div>${html_hide_hover_link(x_href(ast, context))}${out}</div>`;
  },
  [OUTPUT_FORMAT_ID]: function(ast, context, out) {
    return out;
  },
}
